Browse Source

feat: project from excel

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/765/head
Pranav C 3 years ago
parent
commit
63d8f4a164
  1. 105
      packages/nc-gui/components/import/dropOrSelectFile.vue
  2. 24
      packages/nc-gui/components/import/dropOrSelectFileModal.vue
  3. 125
      packages/nc-gui/components/import/excelImport.vue
  4. 8
      packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js
  5. 16
      packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js
  6. 20
      packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js
  7. 0
      packages/nc-gui/components/import/templateParsers/TemplateGenerator.js
  8. 6
      packages/nc-gui/components/project/spreadsheet/components/csvExportImport.vue
  9. 11
      packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue
  10. 17
      packages/nc-gui/components/templates/editor.vue
  11. 28
      packages/nc-gui/pages/projects/index.vue
  12. 11
      packages/nocodb/src/lib/sqlMgr/SqlMgr.ts
  13. 1
      packages/nocodb/src/lib/utils/projectAcl.ts

105
packages/nc-gui/components/import/dropOrSelectFile.vue

@ -0,0 +1,105 @@
<template>
<div class="pa-4">
<div
class="nc-droppable d-flex align-center justify-center flex-column"
:style="{
background : dragOver ? '#7774' : ''
}"
@click="$refs.file.click()"
@drop.prevent="dropHandler"
@dragover.prevent="dragOver = true"
@dragenter.prevent="dragOver = true"
@dragexit="dragOver = false"
@dragleave="dragOver = false"
@dragend="dragOver = false"
>
<v-icon size="50" color="grey">
mdi-file-plus-outline
</v-icon>
<p class="title grey--text mb-1 mt-2">
Select {{ text }} file to Upload
</p>
<p class="grey--text ">
or drag and drop {{ text }} file
</p>
</div>
<input
ref="file"
class="nc-excel-import-input"
type="file"
style="display: none"
:accept="accept"
@change="_change($event)"
>
</div>
</template>
<script>
export default {
name: 'DropOrSelectFile',
props: {
accept: String,
text: String
},
data() {
return {
dragOver: false
}
},
computed: {
dropOrUpload: {
set(v) {
this.$emit('input', v)
},
get() {
return this.value
}
}
},
methods: {
_change(event) {
const files = event.target.files
if (files && files[0]) {
this.$emit('file', files[0])
event.target.value = ''
}
},
dropHandler(ev) {
this.dragOver = false
let file
if (ev.dataTransfer.items) {
// Use DataTransferItemList interface to access the file(s)
if (ev.dataTransfer.items.length && ev.dataTransfer.items[0].kind === 'file') {
file = ev.dataTransfer.items[0].getAsFile()
}
} else if (ev.dataTransfer.files.length) {
file = ev.dataTransfer.files[0]
}
if (this.accept && !this.accept.split(',').some(ext => file.name.endsWith(ext.trim()))) {
return this.$toast.error(`Dropped file is not an accepted file type. The accepted file types are ${this.accept}!`).goAway(3000)
}
if (file) {
this.$emit('file', file)
}
},
dragOverHandler(ev) {
console.log('File(s) in drop zone')
// Prevent default behavior (Prevent file from being opened)
ev.preventDefault()
}
}
}
</script>
<style scoped>
.nc-droppable {
width: 100%;
min-height: 200px;
border-radius: 4px;
border: 2px dashed var(--v-textColor-lighten5);
}
</style>

24
packages/nc-gui/components/import/dropOrSelectFileModal.vue

@ -1,7 +1,7 @@
<template>
<v-dialog v-model="dropOrUpload" max-width="600">
<v-card max-width="600">
<div class="pa-4">
<!-- <div class="pa-4">
<div
class="nc-droppable d-flex align-center justify-center flex-column"
:style="{
@ -18,10 +18,10 @@
<v-icon size="50" color="grey">
mdi-file-plus-outline
</v-icon>
<p class="title grey--text mb-1 mt-2">
<p class="title grey&#45;&#45;text mb-1 mt-2">
Select {{ text }} file to Upload
</p>
<p class="grey--text ">
<p class="grey&#45;&#45;text ">
or drag and drop {{ text }} file
</p>
</div>
@ -33,15 +33,23 @@
:accept="accept"
@change="_change($event)"
>
</div>
</div>-->
<drop-or-select-file
:accept="accept"
:text="text"
v-on="$listeners"
/>
</v-card>
</v-dialog>
</template>
<script>
import DropOrSelectFile from '~/components/import/dropOrSelectFile'
export default {
name: 'DropOrSelectFileModal',
components: { DropOrSelectFile },
props: {
value: Boolean,
accept: String,
@ -63,7 +71,7 @@ export default {
}
},
methods: {
_change(event) {
/* _change(event) {
const files = event.target.files
if (files && files[0]) {
this.$emit('file', files[0])
@ -94,17 +102,17 @@ export default {
// Prevent default behavior (Prevent file from being opened)
ev.preventDefault()
}
} */
}
}
</script>
<style scoped>
.nc-droppable {
/*.nc-droppable {
width: 100%;
min-height: 200px;
border-radius: 4px;
border: 2px dashed var(--v-textColor-lighten5);
}
}*/
</style>

125
packages/nc-gui/components/import/excelImport.vue

@ -2,7 +2,22 @@
<div class="pt-10">
<v-dialog v-model="dropOrUpload" max-width="600">
<v-card max-width="600">
<div class="pa-4">
<v-tabs height="30">
<v-tab>
<v-icon small class="mr-1">
mdi-file-upload-outline
</v-icon>
<span class="caption text-capitalize">Upload</span>
</v-tab>
<v-tab>
<v-icon small class="mr-1">
mdi-link-variant
</v-icon>
<span class="caption text-capitalize">URL</span>
</v-tab>
<v-tab-item>
<div class="nc-excel-import-tab-item ">
<div
class="nc-droppable d-flex align-center justify-center flex-column"
:style="{
@ -16,17 +31,52 @@
@dragleave="dragOver = false"
@dragend="dragOver = false"
>
<v-icon size="50" color="grey">
<x-icon :color="['primary','grey']" size="50">
mdi-file-plus-outline
</v-icon>
<p class="title grey--text mb-1 mt-2">
Select Files to Upload
</x-icon>
<p class="title mb-1 mt-2">
Select File to Upload
</p>
<p class="grey--text ">
or drag and drop files
<p class="grey--text mb-1">
or drag and drop file
</p>
<p class="caption grey--text">
Supported: .xls, .xlsx, .xlsm
</p>
</div>
</div>
</v-tab-item>
<v-tab-item>
<div class="nc-excel-import-tab-item align-center">
<div class="pa-4 d-100 h-100">
<v-form ref="form" v-model="valid">
<div class="d-flex">
<v-text-field
v-model="url"
hide-details="auto"
type="url"
label="Enter excel file url"
class="caption"
outlined
dense
:rules="[v => !!v || 'Required']"
/>
<v-btn class="ml-3" color="primary" @click="loadUrl">
Load
</v-btn>
</div>
</v-form>
</div>
</div>
</v-tab-item>
</v-tabs>
<!-- <div class="my-4 text-center grey&#45;&#45;text">-->
<!-- OR-->
<!-- </div>-->
<!-- <drop-or-select-file />-->
</v-card>
</v-dialog>
@ -37,7 +87,7 @@
class="nc-excel-import-input"
type="file"
style="display: none"
accept=".xlsx, .xls"
accept=".xlsx, .xls, .xlsm"
@change="_change($event)"
>
<v-btn
@ -59,7 +109,7 @@
<v-dialog v-if="templateData" :value="true">
<v-card>
<template-editor :template-data.sync="templateData">
<template-editor :project-template.sync="templateData">
<template #toolbar>
<v-spacer />
<create-project-from-template-btn
@ -95,9 +145,10 @@
<script>
// import XLSX from 'xlsx'
import ExcelTemplateAdapter from '~/components/import/ExcelTemplateAdapter'
import TemplateEditor from '~/components/templates/editor'
import CreateProjectFromTemplateBtn from '~/components/templates/createProjectFromTemplateBtn'
import ExcelUrlTemplateAdapter from '~/components/import/templateParsers/ExcelUrlTemplateAdapter'
import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter'
export default {
name: 'ExcelImport',
@ -108,11 +159,13 @@ export default {
},
data() {
return {
valid: null,
templateData: null,
importData: null,
dragOver: false,
loaderMessage: null,
progress: null
progress: null,
url: ''
}
},
computed: {
@ -150,12 +203,9 @@ export default {
this.dropOrUpload = false
const reader = new FileReader()
reader.onload = (e) => {
reader.onload = async(e) => {
const ab = e.target.result
const templateGenerator = new ExcelTemplateAdapter(file.name, ab)
templateGenerator.parse()
this.templateData = templateGenerator.getTemplate()
this.importData = templateGenerator.getData()
await this.parseAndExtractData('file', ab, file.name)
this.loaderMessage = null
clearInterval(int)
}
@ -170,6 +220,23 @@ export default {
}
reader.readAsArrayBuffer(file)
},
async parseAndExtractData(type, val, name) {
let templateGenerator
switch (type) {
case 'file':
templateGenerator = new ExcelTemplateAdapter(name, val)
break
case 'url':
templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store)
break
}
await templateGenerator.init()
templateGenerator.parse()
this.templateData = templateGenerator.getTemplate()
this.importData = templateGenerator.getData()
},
dropHandler(ev) {
console.log('File(s) dropped')
let file
@ -194,6 +261,22 @@ export default {
// Prevent default behavior (Prevent file from being opened)
ev.preventDefault()
},
async loadUrl() {
if (!this.$refs.form.validate()) { return }
this.loaderMessage = 'Loading excel file from url'
let i = 0
const int = setInterval(() => {
this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}`
}, 1000)
this.dropOrUpload = false
await this.parseAndExtractData('url', this.url, '')
clearInterval(int)
this.loaderMessage = null
}
}
@ -205,6 +288,14 @@ export default {
width: 100%;
min-height: 200px;
border-radius: 4px;
border: 2px dashed var(--v-textColor-lighten5);
border: 2px dashed #ddd;
}
.nc-excel-import-tab-item{
min-height: 400px;
padding: 20px;
display: flex;
align-items: stretch;
width:100%;
}
</style>

8
packages/nc-gui/components/import/CSVTemplateAdapter.js → packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js

@ -1,11 +1,11 @@
import Papaparse from 'papaparse'
import TemplateGenerator from '~/components/import/TemplateGenerator'
import TemplateGenerator from '~/components/import/templateParsers/TemplateGenerator'
export default class CSVTemplateAdapter extends TemplateGenerator {
constructor(name, data) {
super()
this.name = name
this.csv = Papaparse.parse(data, { header: true })
this.csvData = data
this.project = {
title: this.name,
tables: []
@ -13,6 +13,10 @@ export default class CSVTemplateAdapter extends TemplateGenerator {
this.data = {}
}
async init() {
this.csv = Papaparse.parse(this.csvData, { header: true })
}
parseData() {
this.columns = this.csv.meta.fields
this.data = this.csv.data

16
packages/nc-gui/components/import/ExcelTemplateAdapter.js → packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@ -1,6 +1,6 @@
import XLSX from 'xlsx'
import TemplateGenerator from '~/components/import/TemplateGenerator'
import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes'
import TemplateGenerator from '~/components/import/templateParsers/TemplateGenerator'
const excelTypeToUidt = {
d: UITypes.DateTime,
@ -13,7 +13,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
constructor(name, ab) {
super()
this.name = name
this.wb = XLSX.read(new Uint8Array(ab), { type: 'array' })
this.excelData = ab
this.project = {
title: this.name,
tables: []
@ -21,6 +21,10 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
this.data = {}
}
async init() {
this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array' })
}
parse() {
for (let i = 0; i < this.wb.SheetNames.length; i++) {
const sheet = this.wb.SheetNames[i]
@ -31,10 +35,10 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
for (let col = 0; col < rows[0].length; col++) {
const column = {
cn: rows[0][col]
cn: (rows[0][col] || `field${col + 1}`).replace(/\./, '_')
}
const cellProps = ws[`${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`]
const cellProps = ws[`${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`] || {}
column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText
@ -47,8 +51,8 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
const vals = rows.slice(1).map(r => r[col]).filter(v => v !== null && v !== undefined)
// check column is multi or single select by comparing unique values
if (vals.some(v => v && v.includes(','))) {
const flattenedVals = vals.flatMap(v => v ? v.split(',') : [])
if (vals.some(v => v && v.toString().includes(','))) {
const flattenedVals = vals.flatMap(v => v ? v.toString().split(',') : [])
const uniqueVals = new Set(flattenedVals)
if (flattenedVals.length > uniqueVals.size && uniqueVals.size <= flattenedVals.length / 10) {
column.uidt = UITypes.MultiSelect

20
packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js

@ -0,0 +1,20 @@
import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter'
export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter {
constructor(url, $store) {
const name = url.split('/').pop()
super(name, null)
this.url = url
this.$store = $store
}
async init() {
const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'handleAxiosCall',
[{
url: this.url,
responseType: 'arraybuffer'
}]])
this.excelData = res.data
await super.init()
}
}

0
packages/nc-gui/components/import/TemplateGenerator.js → packages/nc-gui/components/import/templateParsers/TemplateGenerator.js

6
packages/nc-gui/components/project/spreadsheet/components/csvExportImport.vue

@ -74,8 +74,8 @@
import FileSaver from 'file-saver'
import DropOrSelectFileModal from '~/components/import/dropOrSelectFileModal'
import CSVTemplateAdapter from '~/components/import/CSVTemplateAdapter'
import ColumnMappingModal from '~/components/project/spreadsheet/components/columnMappingModal'
import CSVTemplateAdapter from '~/components/import/templateParsers/CSVTemplateAdapter'
export default {
name: 'CsvExportImport',
@ -98,9 +98,9 @@ export default {
methods: {
async onCsvFileSelection(file) {
const reader = new FileReader()
reader.onload = (e) => {
reader.onload = async(e) => {
const templateGenerator = new CSVTemplateAdapter(file.name, e.target.result)
await templateGenerator.init()
templateGenerator.parseData()
this.parsedCsv.columns = templateGenerator.getColumns()
this.parsedCsv.data = templateGenerator.getData()

11
packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue

@ -43,8 +43,8 @@ export default {
mixins: [colors],
props: {
loading: Boolean,
templateData: Object,
importData: Object,
templateData: [Array, Object],
importData: [Array, Object],
loaderMessage: String,
progress: Number
},
@ -122,7 +122,10 @@ export default {
this.$ncApis.setProjectId(projectId)
let total = 0; let progress = 0
await Promise.all(Object.entries(this.importData).map(async([table, data]) => {
console.log(this.importData)
debugger
await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => {
await this.$store.dispatch('meta/ActLoadMeta', {
tn: `${prefix}${table}`, project_id: projectId
})
@ -142,7 +145,7 @@ export default {
}
this.$emit('update:progress', null)
}))
})(v)))
}
}

17
packages/nc-gui/components/templates/editor.vue

@ -571,7 +571,7 @@ export default {
props: {
id: [Number, String],
viewMode: Boolean,
templateData: Object
projectTemplate: Object
},
data: () => ({
loading: false,
@ -620,8 +620,12 @@ export default {
},
updateFilename() {
return this.url && this.url.split('/').pop()
}
},
projectTemplate() {
watch: {
project: {
deep: true,
handler() {
const template = {
...this.project,
tables: (this.project.tables || []).map((t) => {
@ -676,9 +680,8 @@ export default {
})
}
this.$emit('update:templateData', template)
return template
this.$emit('update:projectTemplate', template)
}
}
},
@ -689,8 +692,8 @@ export default {
document.removeEventListener('keydown', this.handleKeyDown)
},
mounted() {
if (this.templateData) {
this.parseTemplate(this.templateData)
if (this.projectTemplate) {
this.parseTemplate(this.projectTemplate)
this.expansionPanel = Array.from({ length: this.project.tables.length }, (_, i) => i)
}
const input = this.$refs.projec && this.$refs.project.$el.querySelector('input')

28
packages/nc-gui/pages/projects/index.vue

@ -210,7 +210,7 @@
"
/>
</v-list-item-title>
</v-list-item>
</v-list-item>-->
<v-divider />
<v-list-item
title
@ -223,7 +223,6 @@
</v-icon>
</v-list-item-icon>
<v-list-item-title>
&lt;!&ndash; Create By Connecting <br>To An External Database &ndash;&gt;
<span
class="caption font-weight-regular"
v-html="
@ -231,7 +230,7 @@
"
/>
</v-list-item-title>
</v-list-item>-->
</v-list-item>
</v-list>
</v-menu>
</template>
@ -262,7 +261,7 @@
'items-per-page-options': [20, -1],
}"
class="pa-4 text-left mx-auto "
style="cursor: pointer"
style="cursor: pointer; max-width: 100%"
>
<template #item="props">
<tr
@ -273,6 +272,7 @@
@click="projectRouteHandler(props.item)"
>
<td data-v-step="2">
<div class="d-flex align-center">
<v-icon
x-small
class="mr-2"
@ -310,13 +310,23 @@
}}
</x-icon>
<span
class="title font-weight-regular"
>{{
<v-tooltip bottom>
<template #activator="{on}">
<div
class="d-inline-block title font-weight-regular"
style="min-width:0; max-width:390px; white-space: nowrap;text-overflow: ellipsis; overflow: hidden"
v-on="on"
>
{{
props.item.title
}}</span>
}}
</div>
</template>
<span class="caption">{{ props.item.title }}</span>
</v-tooltip>
</div>
</td>
<td>
<td style="width:150px;min-width:150px;max-width:150px">
<div
v-if="props.item.allowed && _isUIAllowed('projectActions',true)"
:class="{

11
packages/nocodb/src/lib/sqlMgr/SqlMgr.ts

@ -1019,6 +1019,13 @@ export default class SqlMgr {
};
}
public async handleAxiosCall(apiMeta) {
// t = process.hrtime();
const data = await require('axios')(...apiMeta);
return data.data;
}
public axiosRequestMake(apiMeta) {
if (apiMeta.body) {
try {
@ -1349,6 +1356,10 @@ export default class SqlMgr {
console.log('Within REST_API_CALL handler', args);
result = this.handleApiCall(args.args);
break;
case 'handleAxiosCall':
console.log('Within handleAxiosCall handler', args);
result = this.handleAxiosCall(args.args);
break;
case ToolOps.PROJECT_MIGRATIONS_LIST:
console.log('Within PROJECT_MIGRATIONS_LIST handler', args);

1
packages/nocodb/src/lib/utils/projectAcl.ts

@ -212,6 +212,7 @@ export default {
projectList: true
},
user: {
handleAxiosCall: true,
projectList: true,
testConnection: true,
projectCreateByWeb: true,

Loading…
Cancel
Save