mirror of https://github.com/nocodb/nocodb
Daniel Spaude
2 years ago
108 changed files with 10026 additions and 9525 deletions
@ -0,0 +1,24 @@
|
||||
<script lang="ts" setup> |
||||
import { onBeforeUnmount, onMounted, ref, useSmartsheetStoreOrThrow } from '#imports' |
||||
|
||||
const { cellRefs } = useSmartsheetStoreOrThrow() |
||||
|
||||
const el = ref<HTMLTableDataCellElement>() |
||||
|
||||
onMounted(() => { |
||||
cellRefs.value.push(el.value!) |
||||
}) |
||||
|
||||
onBeforeUnmount(() => { |
||||
const index = cellRefs.value.indexOf(el.value!) |
||||
if (index > -1) { |
||||
cellRefs.value.splice(index, 1) |
||||
} |
||||
}) |
||||
</script> |
||||
|
||||
<template> |
||||
<td ref="el"> |
||||
<slot /> |
||||
</td> |
||||
</template> |
||||
@ -0,0 +1,26 @@
|
||||
<script lang="ts" setup> |
||||
import { useExpandedFormDetached } from '#imports' |
||||
|
||||
const { states, close } = useExpandedFormDetached() |
||||
|
||||
const shouldClose = (isVisible: boolean, i: number) => { |
||||
if (!isVisible) close(i) |
||||
} |
||||
</script> |
||||
|
||||
<template> |
||||
<template v-for="(state, i) of states" :key="`expanded-form-${i}`"> |
||||
<LazySmartsheetExpandedForm |
||||
v-model="state.isOpen" |
||||
:row="state.row" |
||||
:load-row="state.loadRow" |
||||
:meta="state.meta" |
||||
:row-id="state.rowId" |
||||
:state="state.state" |
||||
:use-meta-fields="state.useMetaFields" |
||||
:view="state.view" |
||||
@update:model-value="shouldClose($event, i)" |
||||
@cancel="close(i)" |
||||
/> |
||||
</template> |
||||
</template> |
||||
@ -0,0 +1,44 @@
|
||||
import type { TableType, ViewType } from 'nocodb-sdk' |
||||
import { createEventHook, ref, useInjectionState } from '#imports' |
||||
import type { Row } from '~/lib' |
||||
|
||||
interface UseExpandedFormDetachedProps { |
||||
'isOpen'?: boolean |
||||
'row': Row | null |
||||
'state'?: Record<string, any> | null |
||||
'meta': TableType |
||||
'loadRow'?: boolean |
||||
'useMetaFields'?: boolean |
||||
'rowId'?: string |
||||
'view'?: ViewType |
||||
'onCancel'?: Function |
||||
'onUpdate:modelValue'?: Function |
||||
} |
||||
|
||||
const [setup, use] = useInjectionState(() => { |
||||
return ref<UseExpandedFormDetachedProps[]>([]) |
||||
}) |
||||
|
||||
export function useExpandedFormDetached() { |
||||
let states = use()! |
||||
|
||||
if (!states) { |
||||
states = setup() |
||||
} |
||||
|
||||
const closeHook = createEventHook<void>() |
||||
|
||||
const index = ref(-1) |
||||
|
||||
const open = (props: UseExpandedFormDetachedProps) => { |
||||
states.value.push(props) |
||||
index.value = states.value.length - 1 |
||||
} |
||||
|
||||
const close = (i?: number) => { |
||||
states.value.splice(i || index.value, 1) |
||||
if (index.value === i || !i) closeHook.trigger() |
||||
} |
||||
|
||||
return { states, open, close, onClose: closeHook.on } |
||||
} |
||||
@ -1,42 +1,323 @@
|
||||
import { parse } from 'papaparse' |
||||
import TemplateGenerator from './TemplateGenerator' |
||||
|
||||
export default class CSVTemplateAdapter extends TemplateGenerator { |
||||
fileName: string |
||||
project: object |
||||
data: object |
||||
csv: any |
||||
csvData: any |
||||
columns: object |
||||
|
||||
constructor(name: string, data: object) { |
||||
super() |
||||
this.fileName = name |
||||
this.csvData = data |
||||
import type { UploadFile } from 'ant-design-vue' |
||||
import { UITypes } from 'nocodb-sdk' |
||||
import { |
||||
extractMultiOrSingleSelectProps, |
||||
getCheckboxValue, |
||||
getDateFormat, |
||||
isCheckboxType, |
||||
isDecimalType, |
||||
isEmailType, |
||||
isMultiLineTextType, |
||||
isUrlType, |
||||
validateDateWithUnknownFormat, |
||||
} from '#imports' |
||||
|
||||
export default class CSVTemplateAdapter { |
||||
config: Record<string, any> |
||||
source: UploadFile[] | string |
||||
detectedColumnTypes: Record<number, Record<string, number>> |
||||
distinctValues: Record<number, Set<string>> |
||||
headers: Record<number, string[]> |
||||
tables: Record<number, any> |
||||
project: { |
||||
tables: Record<string, any>[] |
||||
} |
||||
|
||||
data: Record<string, any> = {} |
||||
columnValues: Record<number, []> |
||||
|
||||
constructor(source: UploadFile[] | string, parserConfig = {}) { |
||||
this.config = parserConfig |
||||
this.source = source |
||||
this.project = { |
||||
title: this.fileName, |
||||
tables: [], |
||||
} |
||||
this.data = {} |
||||
this.csv = {} |
||||
this.columns = {} |
||||
this.csvData = {} |
||||
this.detectedColumnTypes = {} |
||||
this.distinctValues = {} |
||||
this.headers = {} |
||||
this.columnValues = {} |
||||
this.tables = {} |
||||
} |
||||
|
||||
async init() { |
||||
this.csv = parse(this.csvData, { header: true }) |
||||
async init() {} |
||||
|
||||
initTemplate(tableIdx: number, tn: string, columnNames: string[]) { |
||||
const columnNameRowExist = +columnNames.every((v: any) => v === null || typeof v === 'string') |
||||
const columnNamePrefixRef: Record<string, any> = { id: 0 } |
||||
|
||||
const tableObj: Record<string, any> = { |
||||
table_name: tn, |
||||
ref_table_name: tn, |
||||
columns: [], |
||||
} |
||||
|
||||
this.headers[tableIdx] = [] |
||||
this.tables[tableIdx] = [] |
||||
|
||||
for (const [columnIdx, columnName] of columnNames.entries()) { |
||||
let cn: string = ((columnNameRowExist && columnName.toString().trim()) || `field_${columnIdx + 1}`) |
||||
.replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_') |
||||
.trim() |
||||
while (cn in columnNamePrefixRef) { |
||||
cn = `${cn}${++columnNamePrefixRef[cn]}` |
||||
} |
||||
columnNamePrefixRef[cn] = 0 |
||||
|
||||
this.detectedColumnTypes[columnIdx] = {} |
||||
this.distinctValues[columnIdx] = new Set<string>() |
||||
this.columnValues[columnIdx] = [] |
||||
tableObj.columns.push({ |
||||
column_name: cn, |
||||
ref_column_name: cn, |
||||
meta: {}, |
||||
uidt: UITypes.SingleLineText, |
||||
key: columnIdx, |
||||
}) |
||||
|
||||
this.headers[tableIdx].push(cn) |
||||
this.tables[tableIdx] = tableObj |
||||
} |
||||
} |
||||
|
||||
detectInitialUidt(v: string) { |
||||
if (!isNaN(Number(v)) && !isNaN(parseFloat(v))) return UITypes.Number |
||||
if (validateDateWithUnknownFormat(v)) return UITypes.DateTime |
||||
if (['true', 'True', 'false', 'False', '1', '0', 'T', 'F', 'Y', 'N'].includes(v)) return UITypes.Checkbox |
||||
return UITypes.SingleLineText |
||||
} |
||||
|
||||
parseData() { |
||||
this.columns = this.csv.meta.fields |
||||
this.data = this.csv.data |
||||
detectColumnType(tableIdx: number, data: []) { |
||||
for (let columnIdx = 0; columnIdx < data.length; columnIdx++) { |
||||
// skip null data
|
||||
if (!data[columnIdx]) continue |
||||
const colData: any = [data[columnIdx]] |
||||
const colProps = { uidt: this.detectInitialUidt(data[columnIdx]) } |
||||
// TODO(import): centralise
|
||||
if (isMultiLineTextType(colData)) { |
||||
colProps.uidt = UITypes.LongText |
||||
} else if (colProps.uidt === UITypes.SingleLineText) { |
||||
if (isEmailType(colData)) { |
||||
colProps.uidt = UITypes.Email |
||||
} |
||||
if (isUrlType(colData)) { |
||||
colProps.uidt = UITypes.URL |
||||
} else { |
||||
const checkboxType = isCheckboxType(colData) |
||||
if (checkboxType.length === 1) { |
||||
colProps.uidt = UITypes.Checkbox |
||||
} else { |
||||
if (data[columnIdx] && columnIdx < this.config.maxRowsToParse) { |
||||
this.columnValues[columnIdx].push(data[columnIdx]) |
||||
colProps.uidt = UITypes.SingleSelect |
||||
} |
||||
} |
||||
} |
||||
} else if (colProps.uidt === UITypes.Number) { |
||||
if (isDecimalType(colData)) { |
||||
colProps.uidt = UITypes.Decimal |
||||
} |
||||
} else if (colProps.uidt === UITypes.DateTime) { |
||||
if (data[columnIdx] && columnIdx < this.config.maxRowsToParse) { |
||||
this.columnValues[columnIdx].push(data[columnIdx]) |
||||
} |
||||
} |
||||
if (!(colProps.uidt in this.detectedColumnTypes[columnIdx])) { |
||||
this.detectedColumnTypes[columnIdx] = { |
||||
...this.detectedColumnTypes[columnIdx], |
||||
[colProps.uidt]: 0, |
||||
} |
||||
} |
||||
this.detectedColumnTypes[columnIdx][colProps.uidt] += 1 |
||||
|
||||
if (data[columnIdx]) { |
||||
this.distinctValues[columnIdx].add(data[columnIdx]) |
||||
} |
||||
} |
||||
} |
||||
|
||||
getPossibleUidt(columnIdx: number) { |
||||
const detectedColTypes = this.detectedColumnTypes[columnIdx] |
||||
const len = Object.keys(detectedColTypes).length |
||||
// all records are null
|
||||
if (len === 0) { |
||||
return UITypes.SingleLineText |
||||
} |
||||
// handle numeric case
|
||||
if (len === 2 && UITypes.Number in detectedColTypes && UITypes.Decimal in detectedColTypes) { |
||||
if (detectedColTypes[UITypes.Number] > detectedColTypes[UITypes.Decimal]) { |
||||
return UITypes.Number |
||||
} |
||||
return UITypes.Decimal |
||||
} |
||||
// if there are multiple detected column types
|
||||
// then return either LongText or SingleLineText
|
||||
if (len > 1) { |
||||
if (UITypes.LongText in detectedColTypes) { |
||||
return UITypes.LongText |
||||
} |
||||
return UITypes.SingleLineText |
||||
} |
||||
// otherwise, all records have the same column type
|
||||
return Object.keys(detectedColTypes)[0] |
||||
} |
||||
|
||||
updateTemplate(tableIdx: number) { |
||||
for (let columnIdx = 0; columnIdx < this.headers[tableIdx].length; columnIdx++) { |
||||
const uidt = this.getPossibleUidt(columnIdx) |
||||
if (this.columnValues[columnIdx].length > 0) { |
||||
if (uidt === UITypes.DateTime) { |
||||
const dateFormat: Record<string, number> = {} |
||||
if ( |
||||
this.columnValues[columnIdx].slice(1, this.config.maxRowsToParse).every((v: any) => { |
||||
const isDate = v.split(' ').length === 1 |
||||
if (isDate) { |
||||
dateFormat[getDateFormat(v)] = (dateFormat[getDateFormat(v)] || 0) + 1 |
||||
} |
||||
return isDate |
||||
}) |
||||
) { |
||||
this.tables[tableIdx].columns[columnIdx].uidt = UITypes.Date |
||||
// take the date format with the max occurrence
|
||||
this.tables[tableIdx].columns[columnIdx].meta.date_format = |
||||
Object.keys(dateFormat).reduce((x, y) => (dateFormat[x] > dateFormat[y] ? x : y)) || 'YYYY/MM/DD' |
||||
} else { |
||||
// Datetime
|
||||
this.tables[tableIdx].columns[columnIdx].uidt = uidt |
||||
} |
||||
} else if (uidt === UITypes.SingleSelect || uidt === UITypes.MultiSelect) { |
||||
// assume it is a SingleLineText first
|
||||
this.tables[tableIdx].columns[columnIdx].uidt = UITypes.SingleLineText |
||||
// override with UITypes.SingleSelect or UITypes.MultiSelect if applicable
|
||||
Object.assign(this.tables[tableIdx].columns[columnIdx], extractMultiOrSingleSelectProps(this.columnValues[columnIdx])) |
||||
} else { |
||||
this.tables[tableIdx].columns[columnIdx].uidt = uidt |
||||
} |
||||
delete this.columnValues[columnIdx] |
||||
} else { |
||||
this.tables[tableIdx].columns[columnIdx].uidt = uidt |
||||
} |
||||
} |
||||
} |
||||
|
||||
async _parseTableData(tableIdx: number, source: UploadFile | string, tn: string) { |
||||
return new Promise((resolve, reject) => { |
||||
const that = this |
||||
let steppers = 0 |
||||
if (that.config.shouldImportData) { |
||||
steppers = 0 |
||||
const parseSource = (this.config.importFromURL ? (source as string) : (source as UploadFile).originFileObj)! |
||||
parse(parseSource, { |
||||
download: that.config.importFromURL, |
||||
worker: true, |
||||
skipEmptyLines: 'greedy', |
||||
step(row) { |
||||
steppers += 1 |
||||
if (row && steppers >= +that.config.firstRowAsHeaders + 1) { |
||||
const rowData: Record<string, any> = {} |
||||
for (let columnIdx = 0; columnIdx < that.headers[tableIdx].length; columnIdx++) { |
||||
const column = that.tables[tableIdx].columns[columnIdx] |
||||
const data = (row.data as [])[columnIdx] === '' ? null : (row.data as [])[columnIdx] |
||||
if (column.uidt === UITypes.Checkbox) { |
||||
rowData[column.column_name] = getCheckboxValue(data) |
||||
rowData[column.column_name] = data |
||||
} else if (column.uidt === UITypes.SingleSelect || column.uidt === UITypes.MultiSelect) { |
||||
rowData[column.column_name] = (data || '').toString().trim() || null |
||||
} else { |
||||
// TODO(import): do parsing if necessary based on type
|
||||
rowData[column.column_name] = data |
||||
} |
||||
} |
||||
that.data[tn].push(rowData) |
||||
} |
||||
}, |
||||
complete() { |
||||
resolve(true) |
||||
}, |
||||
error(e: Error) { |
||||
reject(e) |
||||
}, |
||||
}) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
async _parseTableMeta(tableIdx: number, source: UploadFile | string) { |
||||
return new Promise((resolve, reject) => { |
||||
const that = this |
||||
let steppers = 0 |
||||
const tn = ((this.config.importFromURL ? (source as string).split('/').pop() : (source as UploadFile).name) as string) |
||||
.replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_') |
||||
.trim()! |
||||
this.data[tn] = [] |
||||
const parseSource = (this.config.importFromURL ? (source as string) : (source as UploadFile).originFileObj)! |
||||
parse(parseSource, { |
||||
download: that.config.importFromURL, |
||||
worker: true, |
||||
skipEmptyLines: 'greedy', |
||||
step(row) { |
||||
steppers += 1 |
||||
if (row) { |
||||
if (steppers === 1) { |
||||
if (that.config.firstRowAsHeaders) { |
||||
// row.data is header
|
||||
that.initTemplate(tableIdx, tn, row.data as []) |
||||
} else { |
||||
// use dummy column names as header
|
||||
that.initTemplate( |
||||
tableIdx, |
||||
tn, |
||||
[...Array((row.data as []).length)].map((_, i) => `field_${i + 1}`), |
||||
) |
||||
if (that.config.autoSelectFieldTypes) { |
||||
// row.data is data
|
||||
that.detectColumnType(tableIdx, row.data as []) |
||||
} |
||||
} |
||||
} else { |
||||
if (that.config.autoSelectFieldTypes) { |
||||
// row.data is data
|
||||
that.detectColumnType(tableIdx, row.data as []) |
||||
} |
||||
} |
||||
} |
||||
}, |
||||
async complete() { |
||||
that.updateTemplate(tableIdx) |
||||
that.project.tables.push(that.tables[tableIdx]) |
||||
await that._parseTableData(tableIdx, source, tn) |
||||
resolve(true) |
||||
}, |
||||
error(e: Error) { |
||||
reject(e) |
||||
}, |
||||
}) |
||||
}) |
||||
} |
||||
|
||||
async parse() { |
||||
if (this.config.importFromURL) { |
||||
await this._parseTableMeta(0, this.source as string) |
||||
} else { |
||||
await Promise.all( |
||||
(this.source as UploadFile[]).map((file: UploadFile, tableIdx: number) => |
||||
(async (f, idx) => { |
||||
await this._parseTableMeta(idx, f) |
||||
})(file, tableIdx), |
||||
), |
||||
) |
||||
} |
||||
} |
||||
|
||||
getColumns() { |
||||
return this.columns |
||||
return this.project.tables.map((t: Record<string, any>) => t.columns) |
||||
} |
||||
|
||||
getData() { |
||||
return this.data |
||||
} |
||||
|
||||
getTemplate() { |
||||
return this.project |
||||
} |
||||
} |
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,199 @@
|
||||
import sqlite3 from 'sqlite3'; |
||||
import { Readable } from 'stream'; |
||||
|
||||
class EntityMap { |
||||
initialized: boolean; |
||||
cols: string[]; |
||||
db: any; |
||||
|
||||
constructor(...args) { |
||||
this.initialized = false; |
||||
this.cols = args.map((arg) => processKey(arg)); |
||||
this.db = new Promise((resolve, reject) => { |
||||
const db = new sqlite3.Database(':memory:'); |
||||
|
||||
const colStatement = this.cols.length > 0 ? this.cols.join(' TEXT, ') + ' TEXT' : 'mappingPlaceholder TEXT'; |
||||
db.run(`CREATE TABLE mapping (${colStatement})`, (err) => { |
||||
if (err) { |
||||
console.log(err); |
||||
reject(err); |
||||
} |
||||
resolve(db) |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
async init () { |
||||
if (!this.initialized) { |
||||
this.db = await this.db; |
||||
this.initialized = true; |
||||
} |
||||
} |
||||
|
||||
destroy() { |
||||
if (this.initialized && this.db) { |
||||
this.db.close(); |
||||
} |
||||
} |
||||
|
||||
async addRow(row) { |
||||
if (!this.initialized) { |
||||
throw 'Please initialize first!'; |
||||
} |
||||
|
||||
const cols = Object.keys(row).map((key) => processKey(key)); |
||||
const colStatement = cols.map((key) => `'${key}'`).join(', '); |
||||
const questionMarks = cols.map(() => '?').join(', '); |
||||
|
||||
const promises = []; |
||||
|
||||
for (const col of cols.filter((col) => !this.cols.includes(col))) { |
||||
promises.push(new Promise((resolve, reject) => { |
||||
this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => { |
||||
if (err) { |
||||
console.log(err); |
||||
reject(err); |
||||
} |
||||
this.cols.push(col); |
||||
resolve(true); |
||||
}); |
||||
})); |
||||
} |
||||
|
||||
await Promise.all(promises); |
||||
|
||||
const values = Object.values(row).map((val) => { |
||||
if (typeof val === 'object') { |
||||
return `JSON::${JSON.stringify(val)}`; |
||||
} |
||||
return val; |
||||
}); |
||||
|
||||
return new Promise((resolve, reject) => { |
||||
this.db.run(`INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`, values, (err) => { |
||||
if (err) { |
||||
console.log(err); |
||||
reject(err); |
||||
} |
||||
resolve(true); |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
getRow(col, val, res = []): Promise<Record<string, any>> { |
||||
if (!this.initialized) { |
||||
throw 'Please initialize first!'; |
||||
} |
||||
return new Promise((resolve, reject) => { |
||||
col = processKey(col); |
||||
res = res.map((r) => processKey(r)); |
||||
this.db.get(`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping WHERE ${col} = ?`, [val], (err, rs) => { |
||||
if (err) { |
||||
console.log(err); |
||||
reject(err); |
||||
} |
||||
if (rs) { |
||||
rs = processResponseRow(rs); |
||||
} |
||||
resolve(rs) |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
getCount(): Promise<number> { |
||||
if (!this.initialized) { |
||||
throw 'Please initialize first!'; |
||||
} |
||||
return new Promise((resolve, reject) => { |
||||
this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => { |
||||
if (err) { |
||||
console.log(err); |
||||
reject(err); |
||||
} |
||||
resolve(rs.count) |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
getStream(res = []): DBStream { |
||||
if (!this.initialized) { |
||||
throw 'Please initialize first!'; |
||||
} |
||||
res = res.map((r) => processKey(r)); |
||||
return new DBStream(this.db, `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`); |
||||
} |
||||
|
||||
getLimit(limit, offset, res = []): Promise<Record<string, any>[]> { |
||||
if (!this.initialized) { |
||||
throw 'Please initialize first!'; |
||||
} |
||||
return new Promise((resolve, reject) => { |
||||
res = res.map((r) => processKey(r)); |
||||
this.db.all(`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping LIMIT ${limit} OFFSET ${offset}`, (err, rs) => { |
||||
if (err) { |
||||
console.log(err); |
||||
reject(err); |
||||
} |
||||
for (let row of rs) { |
||||
row = processResponseRow(row); |
||||
} |
||||
resolve(rs) |
||||
}); |
||||
}); |
||||
} |
||||
} |
||||
|
||||
class DBStream extends Readable { |
||||
db: any; |
||||
stmt: any; |
||||
sql: any; |
||||
|
||||
constructor(db, sql) { |
||||
super({ objectMode: true}); |
||||
this.db = db; |
||||
this.sql = sql; |
||||
this.stmt = this.db.prepare(this.sql); |
||||
this.on('end', () => this.stmt.finalize()); |
||||
} |
||||
|
||||
_read() { |
||||
let stream = this; |
||||
this.stmt.get(function (err, result) { |
||||
if (err) { |
||||
stream.emit('error', err); |
||||
} else { |
||||
if (result) { |
||||
result = processResponseRow(result); |
||||
} |
||||
stream.push(result || null) |
||||
} |
||||
}); |
||||
} |
||||
} |
||||
|
||||
function processResponseRow(res: any) { |
||||
for (const key of Object.keys(res)) { |
||||
if (res[key] && res[key].startsWith('JSON::')) { |
||||
try { |
||||
res[key] = JSON.parse(res[key].replace('JSON::', '')); |
||||
} catch (e) { |
||||
console.log(e); |
||||
} |
||||
} |
||||
if (revertKey(key) !== key) { |
||||
res[revertKey(key)] = res[key]; |
||||
delete res[key]; |
||||
} |
||||
} |
||||
return res; |
||||
} |
||||
|
||||
function processKey(key) { |
||||
return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`); |
||||
} |
||||
|
||||
function revertKey(key) { |
||||
return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]); |
||||
} |
||||
|
||||
export default EntityMap; |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue