Browse Source

feat: csv export public/private api

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/714/head
Pranav C 3 years ago
parent
commit
599d307a49
  1. 2
      packages/nc-gui/components/project/spreadsheet/components/columnFilterMenu.vue
  2. 130
      packages/nc-gui/components/project/spreadsheet/components/csvExport.vue
  3. 5
      packages/nc-gui/components/project/spreadsheet/public/xcTable.vue
  4. 16
      packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue
  5. 12
      packages/nc-gui/store/sqlMgr.js
  6. 162
      packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts
  7. 208
      packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts
  8. 84
      packages/nocodb/src/lib/noco/meta/NcMetaMgrEE.ts

2
packages/nc-gui/components/project/spreadsheet/components/columnFilterMenu.vue

@ -17,7 +17,7 @@
v-on="on"
>
<v-icon small class="mr-1" color="grey darken-3">
mdi-filter
mdi-filter-outline
</v-icon>
Filter
<v-icon small color="#777">

130
packages/nc-gui/components/project/spreadsheet/components/csvExport.vue

@ -1,27 +1,71 @@
<template>
<v-btn
outlined
class="caption"
small
text
@click="exportCsv"
<v-menu
open-on-hover
bottom
offset-y
>
Export
</v-btn>
<template #activator="{on}">
<v-btn
outlined
class="caption"
small
text
v-on="on"
>
<v-icon small color="#777">
mdi-flash-outline
</v-icon>
Actions
<v-icon small color="#777">
mdi-menu-down
</v-icon>
</v-btn>
</template>
<v-list dense>
<v-list-item
dense
@click="exportCsv"
>
<v-list-item-title>
<v-icon small class="mr-1">
mdi-download-outline
</v-icon>
<span class="caption">
Download as CSV
</span>
</v-list-item-title>
</v-list-item>
<v-list-item
dense
@click="comingSoon"
>
<v-list-item-title>
<v-icon small class="mr-1" color="grey">
mdi-upload-outline
</v-icon>
<span class="caption grey--text">
Upload CSV
</span>
</v-list-item-title>
</v-list-item>
</v-list>
</v-menu>
</template>
<script>
import Papaparse from 'papaparse'
// import Papaparse from 'papaparse'
import FileSaver from 'file-saver'
export default {
name: 'CsvExport',
props: {
data: Array,
meta: Object,
nodes: Object,
availableColumns: Array
selectedView: Object,
publicViewId: String
},
methods: {
@ -100,37 +144,55 @@ export default {
}))
},
async exportCsv() {
// const fields = this.availableColumns.map(c => c._cn)
// const blob = new Blob([Papaparse.unparse(await this.extractCsvData())], { type: 'text/plain;charset=utf-8' })
let offset = 0
let c = 1
try {
const data = await this.$store.dispatch('sqlMgr/ActSqlOp', [
{
dbAlias: this.nodes.dbAlias,
env: '_noco'
},
'xcExportAsCsv',
{
query: {},
model_name: this.meta.tn
},
null,
{
responseType: 'blob'
while (!isNaN(offset) && offset > -1) {
const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [
this.publicViewId
? null
: {
dbAlias: this.nodes.dbAlias,
env: '_noco'
},
this.publicViewId ? 'sharedViewExportAsCsv' : 'xcExportAsCsv',
{
query: { offset },
...(this.publicViewId
? {
view_id: this.publicViewId
}
: {
view_name: this.selectedView.title,
model_name: this.meta.tn
})
},
null,
{
responseType: 'blob'
},
null,
true
])
const data = res.data
offset = +res.headers['nc-export-offset']
const blob = new Blob([data], { type: 'text/plain;charset=utf-8' })
FileSaver.saveAs(blob, `${this.meta._tn}_exported_${c++}.csv`)
if (offset > -1) {
this.$toast.info('Downloading more files').goAway(3000)
} else {
this.$toast.success('Successfully exported all table data').goAway(3000)
}
])
// const url = window.URL.createObjectURL(new Blob([data], { type: 'application/zip' }))
// const link = document.createElement('a')
// link.href = url
// link.setAttribute('download', 'meta.zip') // or any other extension
// document.body.appendChild(link)
// link.click()
const blob = new Blob([data], { type: 'text/plain;charset=utf-8' })
FileSaver.saveAs(blob, `${this.meta._tn}_exported.csv`)
this.$toast.success('Successfully exported metadata').goAway(3000)
}
} catch (e) {
this.$toast.error(e.message).goAway(3000)
}
}
}
}

5
packages/nc-gui/components/project/spreadsheet/public/xcTable.vue

@ -91,6 +91,8 @@
<column-filter-menu v-model="filters" :field-list="realFieldList" />
<csv-export :public-view-id="$route.params.id" :meta="meta" />
<!-- <v-menu>
<template #activator="{ on, attrs }">
<v-icon
@ -194,11 +196,12 @@ import SortListMenu from '../components/sortListMenu'
import ColumnFilterMenu from '../components/columnFilterMenu'
import XcGridView from '../views/xcGridView'
import { SqlUI } from '@/helpers/sqlUi'
import CsvExport from '~/components/project/spreadsheet/components/csvExport'
// import ExpandedForm from "../expandedForm";
export default {
name: 'XcTable',
components: { XcGridView, ColumnFilterMenu, SortListMenu, FieldsMenu },
components: { CsvExport, XcGridView, ColumnFilterMenu, SortListMenu, FieldsMenu },
mixins: [spreadsheet],
props: {
env: String,

16
packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue

@ -61,14 +61,6 @@
<v-spacer class="h-100" @dblclick="debug=true" />
<csv-export
:available-columns="availableColumns"
:data="data"
:meta="meta"
:nodes="nodes"
class="mr-1"
/>
<template v-if="!isForm">
<debug-metas v-if="debug" class="mr-3" />
<v-tooltip bottom>
@ -146,6 +138,14 @@
:field-list="[...realFieldList, ...formulaFieldList]"
dense
/>
<csv-export
:meta="meta"
:nodes="nodes"
:selected-view="selectedView"
class="mr-1"
/>
<v-tooltip
v-if="_isUIAllowed('table-delete')"
bottom

12
packages/nc-gui/store/sqlMgr.js

@ -354,7 +354,7 @@ export const actions = {
}
},
async ActSqlOp({ commit, state, rootState, dispatch }, [args, op, opArgs, cusHeaders, cusAxiosOptions, queryParams]) {
async ActSqlOp({ commit, state, rootState, dispatch }, [args, op, opArgs, cusHeaders, cusAxiosOptions, queryParams, returnResponse]) {
const params = {}
if (this.$router.currentRoute && this.$router.currentRoute.params && this.$router.currentRoute.params.project_id) {
params.project_id = this.$router.currentRoute.params.project_id
@ -370,7 +370,7 @@ export const actions = {
if (cusHeaders) {
Object.assign(headers, cusHeaders)
}
const data = (await this.$axios({
const res = (await this.$axios({
url: '?q=sqlOp_' + op,
baseURL: `${this.$axios.defaults.baseURL}/dashboard`,
data: { api: op, ...args, ...params, args: opArgs },
@ -379,7 +379,9 @@ export const actions = {
params: (args && args.query) || {},
...(cusAxiosOptions || {})
})).data
}))
const data = res.data
// clear meta cache on relation create/delete
// todo: clear only necessary metas
@ -401,6 +403,10 @@ export const actions = {
}
}
if (returnResponse) {
return res
}
return data
} catch (e) {
const err = new Error(e.response.data.msg)

162
packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts

@ -2414,84 +2414,119 @@ class BaseModelSql extends BaseModel {
}, []);
}
public async extractCsvData(args: any = {}) {
const rows = await this.nestedList(args);
// todo: optimize
public async extractCsvData(args: any = {}, fields = null) {
const defaultNestedQueryParams = { ...this.defaultNestedQueryParams };
// // get all nested props by default
// for (const key of Object.keys(defaultNestedQueryParams)) {
// if (key.indexOf('fields') > -1) {
// defaultNestedQueryParams[key] = '*';
// }
// }
let offset = +args.offset || 0;
const limit = 100;
// const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 500;
const csvRows = [];
const startTime = process.hrtime();
let elapsed, temp;
for (
elapsed = 0;
elapsed < timeout;
offset += limit,
temp = process.hrtime(startTime),
elapsed = temp[0] * 1000 + temp[1] / 1000000
) {
const rows = await this.nestedList({
...defaultNestedQueryParams,
...args,
offset,
limit
});
if (!rows?.length) {
offset = -1;
break;
}
for (const row of rows) {
const csvRow = {};
for (const row of rows) {
const csvRow = {};
for (const column of this.columns) {
if (column._cn in row) {
csvRow[column._cn] = this.serializeCellValue({
value: row[column._cn],
column
});
for (const column of this.columns) {
if (column._cn in row) {
csvRow[column._cn] = this.serializeCellValue({
value: row[column._cn],
column
});
}
}
}
for (const vColumn of this.virtualColumns) {
if (vColumn._cn in row && !vColumn.bt && !vColumn.hm && !vColumn.mm) {
if (vColumn.lk) {
} else {
csvRow[vColumn._cn] = row[vColumn._cn];
for (const vColumn of this.virtualColumns) {
if (vColumn._cn in row && !vColumn.bt && !vColumn.hm && !vColumn.mm) {
if (vColumn.lk) {
} else {
csvRow[vColumn._cn] = row[vColumn._cn];
}
}
}
}
for (const [prop, col] of Object.entries(this.nestedProps)) {
const refModel = this._nestedPropsModels[prop];
for (const [prop, col] of Object.entries(this.nestedProps)) {
const refModel = this._nestedPropsModels[prop];
const mapPropFn = (alias, colAlias) => {
if (Array.isArray(row[prop])) {
csvRow[alias] = row[prop].map(r =>
refModel.serializeCellValue({
value: r[colAlias],
const mapPropFn = (alias, colAlias) => {
if (Array.isArray(row[prop])) {
csvRow[alias] = row[prop].map(r =>
refModel.serializeCellValue({
value: r[colAlias],
columnName: colAlias
})
);
} else if (row[prop]) {
csvRow[alias] = refModel.serializeCellValue({
value: row?.[prop]?.[colAlias],
columnName: colAlias
})
);
} else if (row[prop]) {
csvRow[alias] = refModel.serializeCellValue({
value: row[colAlias],
columnName: colAlias
});
}
};
if (prop in row) {
// todo: optimize
for (const vColumn of this.virtualColumns) {
if (vColumn.lk) {
if (
col.hm &&
vColumn.lk.type === 'hm' &&
col.hm.tn === vColumn.lk.ltn
) {
mapPropFn(col._cn, vColumn.lk._lcn);
} else if (
col.mm &&
vColumn.lk.type === 'mm' &&
col.mm.rtn === vColumn.lk.ltn
) {
mapPropFn(col._cn, vColumn.lk._lcn);
}
if (
col.bt &&
vColumn.lk.type === 'bt' &&
col.bt.rtn === vColumn.lk.ltn
) {
mapPropFn(col._cn, vColumn.lk._lcn);
});
}
};
if (prop in row) {
// todo: optimize
for (const vColumn of this.virtualColumns) {
if (vColumn.lk) {
if (
col.hm &&
vColumn.lk.type === 'hm' &&
col.hm.tn === vColumn.lk.ltn
) {
mapPropFn(vColumn._cn, vColumn.lk._lcn);
} else if (
col.mm &&
vColumn.lk.type === 'mm' &&
col.mm.rtn === vColumn.lk.ltn
) {
mapPropFn(vColumn._cn, vColumn.lk._lcn);
}
if (
col.bt &&
vColumn.lk.type === 'bt' &&
col.bt.rtn === vColumn.lk.ltn
) {
mapPropFn(vColumn._cn, vColumn.lk._lcn);
}
}
}
}
mapPropFn(col._cn, refModel.primaryColAlias);
}
mapPropFn(col._cn, refModel.primaryColAlias);
csvRows.push(csvRow);
}
csvRows.push(csvRow);
}
return Papaparse.unparse(csvRows);
const data = Papaparse.unparse({ fields, data: csvRows });
return { data, offset, elapsed };
}
public serializeCellValue({
@ -2506,7 +2541,7 @@ class BaseModelSql extends BaseModel {
return value;
}
const column =
args.column || this.columns.find(c => c._cn === this.columnToAlias);
args.column || this.columns.find(c => c._cn === args.columnName);
switch (column?.uidt) {
case 'Attachment': {
@ -2518,7 +2553,8 @@ class BaseModelSql extends BaseModel {
} catch {}
return (data || []).map(
attachment => `${attachment.title}(${encodeURI(attachment.url)})`
attachment =>
`${encodeURI(attachment.title)}(${encodeURI(attachment.url)})`
);
}
default:

208
packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts

@ -5,7 +5,7 @@ import path from 'path';
import archiver from 'archiver';
import axios from 'axios';
import bodyParser from 'body-parser';
import { Handler, Router } from 'express';
import express, { Handler, Router } from 'express';
import extract from 'extract-zip';
import isDocker from 'is-docker';
import multer from 'multer';
@ -1307,6 +1307,9 @@ export default class NcMetaMgr {
case 'sharedViewGet':
result = await this.sharedViewGet(req, args);
break;
case 'sharedViewExportAsCsv':
result = await this.sharedViewExportAsCsv(req, args, res);
break;
case 'sharedViewInsert':
result = await this.sharedViewInsert(req, args);
@ -1333,6 +1336,10 @@ export default class NcMetaMgr {
return next(e);
}
if (typeof result?.cb === 'function') {
return await result.cb();
}
res.json(result);
}
@ -2771,7 +2778,12 @@ export default class NcMetaMgr {
}
protected getDbAlias(args): string {
return args?.dbAlias || args?.args?.dbAlias;
return (
args?.dbAlias ||
args?.args?.dbAlias ||
args?.db_alias ||
args?.args?.db_alias
);
}
protected isProjectRest() {
@ -3822,6 +3834,49 @@ export default class NcMetaMgr {
return { ...sharedViewMeta, ...viewMeta };
}
protected async sharedViewExportAsCsv(_req, args: any, res): Promise<any> {
const sharedViewMeta = await this.xcMeta
.knex('nc_shared_views')
.where({
view_id: args.args.view_id
})
.first();
if (!sharedViewMeta) {
throw new Error('Meta not found');
}
const viewMeta = await this.xcMeta.metaGet(
sharedViewMeta.project_id,
sharedViewMeta.db_alias,
'nc_models',
{
title: sharedViewMeta.view_name
}
);
if (!viewMeta) {
throw new Error('Not found');
}
if (
sharedViewMeta &&
sharedViewMeta.password &&
sharedViewMeta.password !== args.args.password
) {
throw new Error(this.INVALID_PASSWORD_ERROR);
}
return this.xcExportAsCsv(
{
...sharedViewMeta,
args: { ...sharedViewMeta, offset: 0 }
},
_req,
res
);
}
protected async xcAuthHookGet(args: any): Promise<any> {
try {
return await this.xcMeta.metaGet(args.project_id, 'db', 'nc_hooks', {
@ -3944,24 +3999,78 @@ export default class NcMetaMgr {
return { data: { list: procedures } };
}
protected async xcExportAsCsv(args, _req, res) {
protected async xcExportAsCsv(args, _req, res: express.Response) {
const projectId = this.getProjectId(args);
const dbAlias = this.getDbAlias(args);
const apiBuilder = this.app?.projectBuilders
?.find(pb => pb.id === this.getProjectId(args))
?.apiBuilders?.find(ab => ab.dbAlias === this.getDbAlias(args));
?.find(pb => pb.id === projectId)
?.apiBuilders?.find(ab => ab.dbAlias === dbAlias);
const model = apiBuilder?.xcModels?.[args.args.model_name];
const csvData = await model.extractCsvData(args.args.query || {});
const meta = apiBuilder?.getMeta(args.args.model_name);
const selectedView = await this.xcMeta.metaGet(
projectId,
dbAlias,
'nc_models',
{
title: args.args.view_name
}
);
const queryParams = JSON.parse(selectedView.query_params);
const sort = this.serializeSortParam(queryParams);
const csvData = await model.extractCsvData(
{
...(args.args.query || {}),
fields: meta.columns
.filter(c => queryParams?.showFields?.[c._cn])
.map(c => c._cn)
.join(','),
sort: sort,
where: this.serializeToXwhere(queryParams?.filters),
...this.serializeNestedParams(meta, queryParams)
},
// filter only visible columns
Object.entries(queryParams?.showFields || {})
.filter(v => v[1])
.map(v => v[0])
.sort(
(a, b) =>
queryParams?.fieldsOrder?.indexOf(a) -
queryParams?.fieldsOrder?.indexOf(b)
)
);
return {
cb: async () => {
res.set({
'Access-Control-Expose-Headers': 'nc-export-offset',
'nc-export-offset': csvData.offset,
'nc-export-elapsed-time': csvData.elapsed,
'Content-Disposition': `attachment; filename="${args.args.model_name}-export.csv"`
});
res.send(csvData);
res.send(csvData.data);
}
};
}
private serializeSortParam(queryParams) {
const sort = [];
if (queryParams.sortList) {
sort.push(
...(queryParams?.sortList
?.map(sort => {
return sort.field ? `${sort.order}${sort.field}` : '';
})
.filter(Boolean) || [])
);
}
return sort.join(',');
}
// @ts-ignore
protected async xcVisibilityMetaGet(args) {
try {
@ -4787,6 +4896,91 @@ export default class NcMetaMgr {
return result;
}
protected serializeToXwhere(filters) {
// todo: move this logic to a common library
// todo: replace with condition prop
const privateViewWhere = filters?.reduce?.((condition, filt, i) => {
if (!i && !filt.logicOp) {
return condition;
}
if (!(filt.field && filt.op)) {
return condition;
}
condition += i ? `~${filt.logicOp}` : '';
switch (filt.op) {
case 'is equal':
return condition + `(${filt.field},eq,${filt.value})`;
case 'is not equal':
return condition + `~not(${filt.field},eq,${filt.value})`;
case 'is like':
return condition + `(${filt.field},like,%${filt.value}%)`;
case 'is not like':
return condition + `~not(${filt.field},like,%${filt.value}%)`;
case 'is empty':
return condition + `(${filt.field},in,)`;
case 'is not empty':
return condition + `~not(${filt.field},in,)`;
case 'is null':
return condition + `(${filt.field},is,null)`;
case 'is not null':
return condition + `~not(${filt.field},is,null)`;
case '<':
return condition + `(${filt.field},lt,${filt.value})`;
case '<=':
return condition + `(${filt.field},le,${filt.value})`;
case '>':
return condition + `(${filt.field},gt,${filt.value})`;
case '>=':
return condition + `(${filt.field},ge,${filt.value})`;
}
return condition;
}, '');
return privateViewWhere;
}
protected serializeNestedParams(meta, queryParams) {
const nestedParams: any = {
hm: [],
mm: [],
bt: []
};
for (const v of meta.v) {
if (!queryParams?.showFields?.[v._cn]) continue;
if (v.bt || v.lk?.type === 'bt') {
const tn = v.bt?.rtn || v.lk?.rtn;
if (!nestedParams.bt.includes(tn)) nestedParams.bt.push(tn);
if (v.lk) {
const key = `bf${nestedParams.bt.indexOf(tn)}`;
nestedParams[key] =
(nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
}
} else if (v.hm || v.lk?.type === 'hm') {
const tn = v.hm?.tn || v.lk?.tn;
if (!nestedParams.hm.includes(tn)) nestedParams.hm.push(tn);
if (v.lk) {
const key = `hf${nestedParams.hm.indexOf(tn)}`;
nestedParams[key] =
(nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
}
} else if (v.mm || v.lk?.type === 'mm') {
const tn = v.mm?.rtn || v.lk?.rtn;
if (!nestedParams.mm.includes(tn)) nestedParams.mm.push(tn);
if (v.lk) {
const key = `mf${nestedParams.mm.indexOf(tn)}`;
nestedParams[key] =
(nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
}
}
}
nestedParams.mm = nestedParams.mm.join(',');
nestedParams.hm = nestedParams.hm.join(',');
nestedParams.bt = nestedParams.bt.join(',');
return nestedParams;
}
}
export class XCEeError extends Error {

84
packages/nocodb/src/lib/noco/meta/NcMetaMgrEE.ts

@ -169,49 +169,7 @@ export default class NcMetaMgrEE extends NcMetaMgr {
if (req.query.where) {
where += req.query.where;
}
// todo: move this logic to a common library
// todo: replace with condition prop
const privateViewWhere = queryParams?.filters?.reduce?.(
(condition, filt, i) => {
if (!i && !filt.logicOp) {
return condition;
}
if (!(filt.field && filt.op)) {
return condition;
}
condition += i ? `~${filt.logicOp}` : '';
switch (filt.op) {
case 'is equal':
return condition + `(${filt.field},eq,${filt.value})`;
case 'is not equal':
return condition + `~not(${filt.field},eq,${filt.value})`;
case 'is like':
return condition + `(${filt.field},like,%${filt.value}%)`;
case 'is not like':
return condition + `~not(${filt.field},like,%${filt.value}%)`;
case 'is empty':
return condition + `(${filt.field},in,)`;
case 'is not empty':
return condition + `~not(${filt.field},in,)`;
case 'is null':
return condition + `(${filt.field},is,null)`;
case 'is not null':
return condition + `~not(${filt.field},is,null)`;
case '<':
return condition + `(${filt.field},lt,${filt.value})`;
case '<=':
return condition + `(${filt.field},le,${filt.value})`;
case '>':
return condition + `(${filt.field},gt,${filt.value})`;
case '>=':
return condition + `(${filt.field},ge,${filt.value})`;
}
return condition;
},
''
);
const privateViewWhere = this.serializeToXwhere(queryParams?.filters);
if (privateViewWhere) {
where += where ? `~and(${privateViewWhere})` : privateViewWhere;
@ -231,45 +189,7 @@ export default class NcMetaMgrEE extends NcMetaMgr {
}
const fields = meta.columns.map(c => c._cn).join(',');
const nestedParams: any = {
hm: [],
mm: [],
bt: []
};
for (const v of meta.v) {
if (!queryParams?.showFields?.[v._cn]) continue;
if (v.bt || v.lk?.type === 'bt') {
const tn = v.bt?.rtn || v.lk?.rtn;
if (!nestedParams.bt.includes(tn)) nestedParams.bt.push(tn);
if (v.lk) {
const key = `bf${nestedParams.bt.indexOf(tn)}`;
nestedParams[key] =
(nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
}
} else if (v.hm || v.lk?.type === 'hm') {
const tn = v.hm?.tn || v.lk?.tn;
if (!nestedParams.hm.includes(tn)) nestedParams.hm.push(tn);
if (v.lk) {
const key = `hf${nestedParams.hm.indexOf(tn)}`;
nestedParams[key] =
(nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
}
} else if (v.mm || v.lk?.type === 'mm') {
const tn = v.mm?.rtn || v.lk?.rtn;
if (!nestedParams.mm.includes(tn)) nestedParams.mm.push(tn);
if (v.lk) {
const key = `mf${nestedParams.mm.indexOf(tn)}`;
nestedParams[key] =
(nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
}
}
}
nestedParams.mm = nestedParams.mm.join(',');
nestedParams.hm = nestedParams.hm.join(',');
nestedParams.bt = nestedParams.bt.join(',');
const nestedParams = this.serializeNestedParams(meta, queryParams);
return {
model_name: sharedViewMeta.model_name,

Loading…
Cancel
Save