Browse Source

feat: csv import

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/765/head
Pranav C 3 years ago
parent
commit
6dbea600f8
  1. 13
      packages/nc-gui/components/base/shareBase.vue
  2. 28
      packages/nc-gui/components/import/CSVTemplateAdapter.js
  3. 12
      packages/nc-gui/components/import/TemplateGenerator.js
  4. 109
      packages/nc-gui/components/import/dropOrSelectFileModal.vue
  5. 18
      packages/nc-gui/components/import/excelImport.vue
  6. 40
      packages/nc-gui/components/loader.vue
  7. 118
      packages/nc-gui/components/project/spreadsheet/components/columnMappingModal.vue
  8. 72
      packages/nc-gui/components/project/spreadsheet/components/csvExportImport.vue
  9. 5
      packages/nc-gui/components/project/spreadsheet/helpers/uiTypes.js
  10. 6
      packages/nc-gui/components/project/spreadsheet/public/xcTable.vue
  11. 6
      packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue
  12. 4
      packages/nc-gui/layouts/default.vue
  13. 22
      packages/nc-gui/store/loader.js

13
packages/nc-gui/components/base/shareBase.vue

@ -53,6 +53,9 @@
<v-icon @click="navigateToSharedBase">
mdi-open-in-new
</v-icon>
<v-icon @click="generateEmbeddableIframe">
mdi-xml
</v-icon>
</div>
</v-chip>
</div>
@ -120,6 +123,16 @@ export default {
},
navigateToSharedBase() {
window.open(this.url, '_blank')
},
generateEmbeddableIframe() {
copyTextToClipboard(`<iframe
class="nc-embed"
src="${this.url}?embedded"
frameborder="0"
width="100%"
height="700"
style="background: transparent; "></iframe>`)
this.$toast.success('Copied embeddable html code!').goAway(3000)
}
}

28
packages/nc-gui/components/import/CSVTemplateAdapter.js

@ -0,0 +1,28 @@
import Papaparse from 'papaparse'
import TemplateGenerator from '~/components/import/TemplateGenerator'
export default class CSVTemplateAdapter extends TemplateGenerator {
constructor(name, data) {
super()
this.name = name
this.csv = Papaparse.parse(data, { header: true })
this.project = {
title: this.name,
tables: []
}
this.data = {}
}
parseData() {
this.columns = this.csv.meta.fields
this.data = this.csv.data
}
getColumns() {
return this.columns
}
getData() {
return this.data
}
}

12
packages/nc-gui/components/import/TemplateGenerator.js

@ -3,6 +3,18 @@ export default class TemplateGenerator {
throw new Error('\'parse\' method is not implemented')
}
parseData() {
throw new Error('\'parseData\' method is not implemented')
}
parseTemplate() {
throw new Error('\'parseTemplate\' method is not implemented')
}
getColumns() {
throw new Error('\'getColumns\' method is not implemented')
}
getTemplate() {
throw new Error('\'getTemplate\' method is not implemented')
}

109
packages/nc-gui/components/import/dropOrSelectFileModal.vue

@ -0,0 +1,109 @@
<template>
<v-dialog v-model="dropOrUpload" max-width="600">
<v-card max-width="600">
<div class="pa-4">
<div
class="nc-droppable d-flex align-center justify-center flex-column"
:style="{
background : dragOver ? '#7774' : ''
}"
@click="$refs.file.click()"
@drop.prevent="dropHandler"
@dragover.prevent="dragOver = true"
@dragenter.prevent="dragOver = true"
@dragexit="dragOver = false"
@dragleave="dragOver = false"
@dragend="dragOver = false"
>
<v-icon size="50" color="grey">
mdi-file-plus-outline
</v-icon>
<p class="title grey--text mb-1 mt-2">
Select {{ text }} file to Upload
</p>
<p class="grey--text ">
or drag and drop {{ text }} file
</p>
</div>
<input
ref="file"
class="nc-excel-import-input"
type="file"
style="display: none"
:accept="accept"
@change="_change($event)"
>
</div>
</v-card>
</v-dialog>
</template>
<script>
export default {
name: 'DropOrSelectFileModal',
props: {
value: Boolean,
accept: String,
text: String
},
data() {
return {
dragOver: false
}
},
computed: {
dropOrUpload: {
set(v) {
this.$emit('input', v)
},
get() {
return this.value
}
}
},
methods: {
_change(file) {
const files = file.target.files
if (files && files[0]) {
this.$emit('file', files[0])
}
},
dropHandler(ev) {
this.dragOver = false
let file
if (ev.dataTransfer.items) {
// Use DataTransferItemList interface to access the file(s)
if (ev.dataTransfer.items.length && ev.dataTransfer.items[0].kind === 'file') {
file = ev.dataTransfer.items[0].getAsFile()
}
} else if (ev.dataTransfer.files.length) {
file = ev.dataTransfer.files[0]
}
if (this.accept && !this.accept.split(',').some(ext => file.name.endsWith(ext.trim()))) {
return this.$toast.error(`Dopped file is not an accepted file type. The accepted file types are ${this.accept}!`).goAway(3000)
}
if (file) {
this.$emit('file', file)
}
},
dragOverHandler(ev) {
console.log('File(s) in drop zone')
// Prevent default behavior (Prevent file from being opened)
ev.preventDefault()
}
}
}
</script>
<style scoped>
.nc-droppable {
width: 100%;
min-height: 200px;
border-radius: 4px;
border: 2px dashed var(--v-textColor-lighten5);
}
</style>

18
packages/nc-gui/components/import/excelImport.vue

@ -1,10 +1,10 @@
<template>
<div>
<v-dialog max-width="600" :value="dropOrUpload">
<div class="pt-10">
<v-dialog v-model="dropOrUpload" max-width="600">
<v-card max-width="600">
<div class="pa-4">
<div
class="nc-droppable d-flex align-center justify-center"
class="nc-droppable d-flex align-center justify-center flex-column"
:style="{
background : dragOver ? '#7774' : ''
}"
@ -17,8 +17,14 @@
@dragend="dragOver = false"
>
<v-icon size="50" color="grey">
mdi-upload
mdi-file-plus-outline
</v-icon>
<p class="title grey--text mb-1 mt-2">
Select Files to Upload
</p>
<p class="grey--text ">
or drag and drop files
</p>
</div>
</div>
</v-card>
@ -175,6 +181,10 @@ export default {
} else if (ev.dataTransfer.files.length) {
file = ev.dataTransfer.files[0]
}
if (file.type !== 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' && file.type !== 'application/vnd.ms-excel') {
return this.$toast.error('Dropped file is not an accepted file type. The accepted file types are .xlsx,.xls!').goAway(3000)
}
if (file) {
this._file(file)
}

40
packages/nc-gui/components/loader.vue

@ -0,0 +1,40 @@
<template>
<v-overlay :value="message" z-index="99999" opacity=".9">
<div class="d-flex flex-column align-center">
<v-progress-circular
v-if="progress !== null "
:rotate="360"
:size="100"
:width="15"
:value="progress"
>
{{ progress }}%
</v-progress-circular>
<v-progress-circular v-else indeterminate size="100" width="15" class="mb-10" />
<span class="title">{{ message }}</span>
</div>
</v-overlay>
</template>
<script>
export default {
name: 'Loader',
props: {
// message: String,
// progress: Number
},
computed: {
message() {
return this.$store.state.loader.message
},
progress() {
return this.$store.state.loader.progress
}
}
}
</script>
<style scoped>
</style>

118
packages/nc-gui/components/project/spreadsheet/components/columnMappingModal.vue

@ -0,0 +1,118 @@
<template>
<v-dialog v-model="visible" max-width="800px">
<v-card>
<v-card-actions>
<v-card-title>
Table : {{ meta._tn }}
</v-card-title>
<v-spacer />
<v-btn color="primary" large @click="$emit('import',mappings)">
<v-icon small class="mr-1">
mdi-database-import-outline
</v-icon> Import
</v-btn>
</v-card-actions>
<v-divider />
<v-container fluid>
<v-simple-table dense style="position:relative;">
<thead>
<tr>
<th />
<th style="width:45%" class="grey--text">
Source column
</th>
<th style="width:45%" class="grey--text">
Destination column
</th>
</tr>
</thead>
<tbody>
<tr v-for="(r,i) in mappings" :key="i">
<td>
<v-checkbox v-model="r.enabled" class="mt-0" dense hide-details />
</td><td class="caption" style="width:45%">
<div :title="r.sourceCn" style="">
{{ r.sourceCn }}
</div>
</td><td style="width:45%">
<v-select
v-model="r.destCn"
class="caption"
dense
hide-details
:items="meta.columns"
item-text="_cn"
item-value="_cn"
>
<template #selection="{item}">
<v-icon small class="mr-1">
{{ getIcon(item.uidt) }}
</v-icon>
{{ item._cn }}
</template>
<template #item="{item}">
<v-icon small class="mr-1">
{{ getIcon(item.uidt) }}
</v-icon>
<span class="caption"> {{ item._cn }}</span>
</template>
</v-select>
</td>
</tr>
</tbody>
</v-simple-table>
</v-container>
</v-card>
</v-dialog>
</template>
<script>
import { getUIDTIcon } from '~/components/project/spreadsheet/helpers/uiTypes'
export default {
name: 'ColumnMappingModal',
props: {
meta: Object,
importDataColumns: Array,
value: Boolean
},
data() {
return {
mappings: []
}
},
computed: {
visible: {
get() {
return this.value
},
set(v) {
this.$emit('input', v)
}
}
},
mounted() {
this.mapDefaultColumns()
},
methods: {
mapDefaultColumns() {
this.mappings = []
for (const col of this.importDataColumns) {
const o = { sourceCn: col, enabled: true }
const tableColumn = this.meta.columns.find(c => c._cn === col)
if (tableColumn) {
o.destCn = tableColumn._cn
}
this.mappings.push(o)
}
},
getIcon(uidt) {
return getUIDTIcon(uidt) || 'mdi-alpha-v-circle-outline'
}
}
}
</script>
<style scoped>
</style>

72
packages/nc-gui/components/project/spreadsheet/components/csvExport.vue → packages/nc-gui/components/project/spreadsheet/components/csvExportImport.vue

@ -1,4 +1,5 @@
<template>
<div>
<v-menu
open-on-hover
bottom
@ -39,28 +40,40 @@
</v-list-item>
<v-list-item
dense
@click="comingSoon"
@click="importModal = true"
>
<v-list-item-title>
<v-icon small class="mr-1" color="grey">
<v-icon small class="mr-1" color="">
mdi-upload-outline
</v-icon>
<span class="caption grey--text">
<span class="caption ">
Upload CSV
</span>
</v-list-item-title>
</v-list-item>
</v-list>
</v-menu>
<drop-or-select-file-modal v-model="importModal" accept=".csv" text="CSV" @file="onCsvFileSelection" />
<column-mapping-modal
v-if="columnMappingModal && meta"
v-model="columnMappingModal"
:meta="meta"
:import-data-columns="parsedCsv.columns"
@import="importData"
/>
</div>
</template>
<script>
// import Papaparse from 'papaparse'
import FileSaver from 'file-saver'
import DropOrSelectFileModal from '~/components/import/dropOrSelectFileModal'
import CSVTemplateAdapter from '~/components/import/CSVTemplateAdapter'
import ColumnMappingModal from '~/components/project/spreadsheet/components/columnMappingModal'
export default {
name: 'CsvExport',
name: 'CsvExportImport',
components: { ColumnMappingModal, DropOrSelectFileModal },
props: {
meta: Object,
nodes: Object,
@ -68,8 +81,30 @@ export default {
publicViewId: String,
queryParams: Object
},
data() {
return {
importModal: false,
columnMappingModal: false,
parsedCsv: {}
}
},
methods: {
async onCsvFileSelection(file) {
const reader = new FileReader()
reader.onload = (e) => {
const templateGenerator = new CSVTemplateAdapter(file.name, e.target.result)
templateGenerator.parseData()
this.parsedCsv.columns = templateGenerator.getColumns()
this.parsedCsv.data = templateGenerator.getData()
this.columnMappingModal = true
this.importModal = false
}
reader.readAsText(file)
},
async extractCsvData() {
return Promise.all(this.data.map(async(r) => {
const row = {}
@ -193,8 +228,33 @@ export default {
} catch (e) {
this.$toast.error(e.message).goAway(3000)
}
}
},
async importData(columnMappings) {
try {
const api = this.$ncApis.get({
table: this.meta.tn
})
const data = this.parsedCsv.data
for (let i = 0, progress = 0; i < data.length; i += 500) {
this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${data.length}`)
this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / data.length))
const batchData = data.slice(i, i + 500).map(row => columnMappings.reduce((res, col) => {
if (col.enabled) {
res[col.destCn] = row[col.sourceCn]
}
return res
}, {}))
await api.insertBulk(batchData)
progress += batchData.length
}
this.columnMappingModal = false
this.$store.commit('loader/MutClear')
this.$toast.success('Successfully imported table data').goAway(3000)
} catch (e) {
this.$toast.error(e.message).goAway(3000)
}
}
}
}

5
packages/nc-gui/components/project/spreadsheet/helpers/uiTypes.js

@ -151,7 +151,10 @@ const uiTypes = [
]
const getUIDTIcon = (uidt) => {
return (uiTypes.find(t => t.name === uidt) || {}).icon
return ([...uiTypes, {
name: 'CreateTime',
icon: 'mdi-calendar-clock'
}].find(t => t.name === uidt) || {}).icon
}
export {

6
packages/nc-gui/components/project/spreadsheet/public/xcTable.vue

@ -91,7 +91,7 @@
<column-filter-menu v-model="filters" :field-list="realFieldList" />
<csv-export :query-params="{...queryParams, showFields}" :public-view-id="$route.params.id" :meta="meta" />
<csv-export-import :query-params="{...queryParams, showFields}" :public-view-id="$route.params.id" :meta="meta" />
<!-- <v-menu>
<template #activator="{ on, attrs }">
@ -197,12 +197,12 @@ import SortListMenu from '../components/sortListMenu'
import ColumnFilterMenu from '../components/columnFilterMenu'
import XcGridView from '../views/xcGridView'
import { SqlUI } from '@/helpers/sqlUi'
import CsvExport from '~/components/project/spreadsheet/components/csvExport'
import CsvExportImport from '~/components/project/spreadsheet/components/csvExportImport'
// import ExpandedForm from "../expandedForm";
export default {
name: 'XcTable',
components: { CsvExport, XcGridView, ColumnFilterMenu, SortListMenu, FieldsMenu },
components: { CsvExportImport, XcGridView, ColumnFilterMenu, SortListMenu, FieldsMenu },
mixins: [spreadsheet],
props: {
env: String,

6
packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue

@ -139,7 +139,7 @@
dense
/>
<csv-export
<csv-export-import
:meta="meta"
:nodes="nodes"
:query-params="{
@ -557,12 +557,12 @@ import ExpandedForm from '@/components/project/spreadsheet/components/expandedFo
import Pagination from '@/components/project/spreadsheet/components/pagination'
import { SqlUI } from '~/helpers/sqlUi'
import ColumnFilter from '~/components/project/spreadsheet/components/columnFilterMenu'
import CsvExport from '~/components/project/spreadsheet/components/csvExport'
import CsvExportImport from '~/components/project/spreadsheet/components/csvExportImport'
export default {
name: 'RowsXcDataTable',
components: {
CsvExport,
CsvExportImport,
FormView,
DebugMetas,
Pagination,

4
packages/nc-gui/layouts/default.vue

@ -554,6 +554,8 @@
</v-btn>
</v-snackbar>
<change-env v-model="showChangeEnv" />
<loader />
</v-app>
<v-app v-else>
<v-overlay>
@ -576,9 +578,11 @@ import { copyTextToClipboard } from '@/helpers/xutils'
import Snackbar from '~/components/snackbar'
import Language from '~/components/utils/language'
import TemplatesModal from '~/components/templates/templatesModal'
import Loader from '~/components/loader'
export default {
components: {
Loader,
TemplatesModal,
ReleaseInfo,
Language,

22
packages/nc-gui/store/loader.js

@ -0,0 +1,22 @@
const state = () => ({
message: null,
progress: null
})
const mutations = {
MutMessage(state, message) {
state.message = message
},
MutProgress(state, progress) {
state.progress = progress
},
MutClear(state) {
state.progress = null
state.message = null
}
}
export {
state,
mutations
}
Loading…
Cancel
Save