Browse Source

Merge pull request #1931 from nocodb/fix/excel-import

fix: excel import
pull/1998/head
աɨռɢӄաօռɢ 3 years ago committed by GitHub
parent
commit
89048d61ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      packages/nc-gui/components/import/excelImport.vue
  2. 6
      packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js
  3. 12
      packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js
  4. 199
      packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue
  5. 1135
      packages/nc-gui/components/templates/editor.vue
  6. 20
      packages/nc-gui/pages/projects/index.vue
  7. 20
      packages/noco-docs/content/en/setup-and-usages/dashboard.md
  8. 18
      packages/nocodb-sdk/src/lib/Api.ts
  9. 2
      packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
  10. 47
      packages/nocodb/src/lib/noco/meta/api/utilApis.ts
  11. 33
      scripts/sdk/swagger.json

4
packages/nc-gui/components/import/excelImport.vue

@ -66,7 +66,7 @@
dense dense
:rules="[v => !!v || $t('general.required') ]" :rules="[v => !!v || $t('general.required') ]"
/> />
<v-btn class="ml-3" color="primary" @click="loadUrl"> <v-btn v-t="['c:project:create:excel:load-url']" class="ml-3" color="primary" @click="loadUrl">
<!--Load--> <!--Load-->
{{ $t('general.load') }} {{ $t('general.load') }}
</v-btn> </v-btn>
@ -266,7 +266,7 @@ export default {
templateGenerator = new ExcelTemplateAdapter(name, val, this.parserConfig) templateGenerator = new ExcelTemplateAdapter(name, val, this.parserConfig)
break break
case 'url': case 'url':
templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig) templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig, this.$api)
break break
} }
await templateGenerator.init() await templateGenerator.init()

6
packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@ -42,7 +42,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
} }
tableNamePrefixRef[tn] = 0 tableNamePrefixRef[tn] = 0
const table = { tn, refTn: tn, columns: [] } const table = { table_name: tn, ref_table_name: tn, columns: [] }
this.data[tn] = [] this.data[tn] = []
const ws = this.wb.Sheets[sheet] const ws = this.wb.Sheets[sheet]
const range = XLSX.utils.decode_range(ws['!ref']) const range = XLSX.utils.decode_range(ws['!ref'])
@ -79,8 +79,8 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
columnNamePrefixRef[cn] = 0 columnNamePrefixRef[cn] = 0
const column = { const column = {
cn, column_name: cn,
refCn: cn ref_column_name: cn
} }
table.columns.push(column) table.columns.push(column)

12
packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js

@ -1,20 +1,22 @@
import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter' import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter'
export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter { export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter {
constructor(url, $store, parserConfig) { constructor(url, $store, parserConfig, $api) {
const name = url.split('/').pop() const name = url.split('/').pop()
super(name, null, parserConfig) super(name, null, parserConfig)
this.url = url this.url = url
this.$api = $api
this.$store = $store this.$store = $store
} }
async init() { async init() {
const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'handleAxiosCall', const data = await this.$api.utils.axiosRequestMake({
[{ apiMeta: {
url: this.url, url: this.url,
responseType: 'arraybuffer' responseType: 'arraybuffer'
}]]) }
this.excelData = res.data })
this.excelData = data.data
await super.init() await super.init()
} }
} }

199
packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue

@ -13,6 +13,7 @@
</template> </template>
<script> <script>
import { SqlUiFactory } from 'nocodb-sdk'
import colors from '~/mixins/colors' import colors from '~/mixins/colors'
export default { export default {
@ -43,7 +44,9 @@ export default {
}, },
data() { data() {
return { return {
localTemplateData: null,
projectCreation: false, projectCreation: false,
tableCreation: false,
loaderMessagesIndex: 0, loaderMessagesIndex: 0,
loaderMessages: [ loaderMessages: [
'Setting up new database configs', 'Setting up new database configs',
@ -69,6 +72,18 @@ export default {
] ]
} }
}, },
watch: {
templateData: {
deep: true,
handler(data) {
this.localTemplateData = JSON.parse(JSON.stringify(data))
}
}
},
created() {
this.localTemplateData = JSON.parse(JSON.stringify(this.templateData))
},
methods: { methods: {
async useTemplate(projectType) { async useTemplate(projectType) {
if (!this.valid) { if (!this.valid) {
@ -76,8 +91,7 @@ export default {
} }
// this.$emit('useTemplate', type) // this.$emit('useTemplate', type)
// this.projectCreation = true
this.projectCreation = true
let interv let interv
try { try {
interv = setInterval(() => { interv = setInterval(() => {
@ -85,111 +99,126 @@ export default {
this.$store.commit('loader/MutMessage', this.loaderMessages[this.loaderMessagesIndex]) this.$store.commit('loader/MutMessage', this.loaderMessages[this.loaderMessagesIndex])
}, 1000) }, 1000)
let projectId, prefix let project
// Not available now
if (this.importToProject) { if (this.importToProject) {
this.$store.commit('loader/MutMessage', 'Importing excel template') // this.$store.commit('loader/MutMessage', 'Importing excel template')
const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [{ // const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [{
// todo: extract based on active // // todo: extract based on active
dbAlias: 'db', // this.nodes.dbAlias, // dbAlias: 'db', // this.nodes.dbAlias,
env: '_noco' // env: '_noco'
}, 'xcModelsCreateFromTemplate', { // }, 'xcModelsCreateFromTemplate', {
template: this.templateData // template: this.templateData
}]) // }])
if (res && res.tables && res.tables.length) { // if (res && res.tables && res.tables.length) {
this.$toast.success(`Imported ${res.tables.length} tables successfully`).goAway(3000) // this.$toast.success(`Imported ${res.tables.length} tables successfully`).goAway(3000)
} else { // } else {
this.$toast.success('Template imported successfully').goAway(3000) // this.$toast.success('Template imported successfully').goAway(3000)
// }
// projectId = this.$route.params.project_id
// prefix = this.$store.getters['project/GtrProjectPrefix']
} else {
// Create an empty project
try {
this.$e("a:project:create:excel");
project = await this.$api.project.create({
title: this.templateData.title,
external: false
})
this.projectCreation = true
} catch (e) {
this.projectCreation = false
this.$toast
.error(await this._extractSdkResponseErrorMsg(e))
.goAway(3000)
} finally {
clearInterval(interv)
} }
projectId = this.$route.params.project_id if (!this.projectCreation) {
prefix = this.$store.getters['project/GtrProjectPrefix'] // failed to create project
} else { return
const result = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'projectCreateByWebWithXCDB', { }
title: this.templateData.title,
projectType, // Create tables
template: this.templateData, try {
excelImport: this.excelImport for (const t of this.localTemplateData.tables) {
}]) // enrich system fields if not provided
projectId = result.id // e.g. id, created_at, updated_at
prefix = result.prefix const systemColumns = SqlUiFactory
await this.$store.dispatch('project/ActLoadProjectInfo') .create({ client: 'sqlite3' })
.getNewTableColumns()
.filter(c => c.column_name != 'title')
const table = await this.$api.dbTable.create(project.id, {
table_name: t.table_name,
title: '',
columns: [...t.columns, ...systemColumns]
})
t.table_title = table.title
}
this.tableCreation = true
} catch (e) {
this.$toast
.error(await this._extractSdkResponseErrorMsg(e))
.goAway(3000)
this.tableCreation = false
} finally {
clearInterval(interv)
}
} }
clearInterval(interv)
if (!this.tableCreation) {
// failed to create table
return
}
// Bulk import data
if (this.importData) { if (this.importData) {
this.$store.commit('loader/MutMessage', 'Importing excel data to project') this.$store.commit('loader/MutMessage', 'Importing excel data to project')
await this.importDataToProject({ projectId, projectType, prefix }) await this.importDataToProject(this.templateData.title)
} }
this.$store.commit('loader/MutMessage', null)
this.projectReloading = false this.$router.push({
if (!this.importToProject) { path: `/nc/${project.id}`,
await this.$router.push({ query: {
path: `/nc/${projectId}`, new: 1
query: { }
new: 1 })
}
})
}
this.$emit('success') this.$emit('success')
} catch (e) { } catch (e) {
console.log(e) console.log(e)
this.$toast.error(e.message).goAway(3000) this.$toast.error(e.message).goAway(3000)
this.$store.commit('loader/MutMessage', null) } finally {
clearInterval(interv) clearInterval(interv)
this.$store.commit('loader/MutMessage', null)
this.projectCreation = false
this.tableCreation = false
} }
this.projectCreation = false
}, },
async importDataToProject({ projectId, projectType, prefix = '' }) { async importDataToProject(projectName) {
// this.$store.commit('project/MutProjectId', projectId)
this.$ncApis.setProjectId(projectId)
let total = 0 let total = 0
let progress = 0 let progress = 0
await Promise.all(this.localTemplateData.tables.map(v => (async(tableMeta) => {
/* await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => { const tableName = tableMeta.table_title
await this.$store.dispatch('meta/ActLoadMeta', { const data = this.importData[tableMeta.ref_table_name]
tn: `${prefix}${table}`, project_id: projectId
})
// todo: get table name properly
const api = this.$ncApis.get({
table: `${prefix}${table}`,
type: projectType
})
total += data.length
for (let i = 0; i < data.length; i += 500) {
this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`)
this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
const batchData = data.slice(i, i + 500)
await api.insertBulk(batchData)
progress += batchData.length
}
this.$store.commit('loader/MutClear')
})(v))) */
await Promise.all(this.templateData.tables.map(v => (async(tableMeta) => {
const table = tableMeta.table_name
const data = this.importData[tableMeta.refTn]
await this.$store.dispatch('meta/ActLoadMeta', {
tn: `${prefix}${table}`, project_id: projectId
})
// todo: get table name properly
const api = this.$ncApis.get({
table: `${prefix}${table}`,
type: projectType
})
total += data.length total += data.length
for (let i = 0; i < data.length; i += 500) { for (let i = 0; i < data.length; i += 500) {
this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`) this.$store.commit('loader/MutMessage', `Importing data to ${projectName}: ${progress}/${total} records`)
this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total)) this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
const batchData = this.remapColNames(data.slice(i, i + 500), tableMeta.columns) const batchData = this.remapColNames(data.slice(i, i + 500), tableMeta.columns)
await api.insertBulk(batchData) await this.$api.dbTableRow.bulkCreate(
'noco',
projectName,
tableName,
batchData
)
progress += batchData.length progress += batchData.length
} }
this.$store.commit('loader/MutClear') this.$store.commit('loader/MutClear')
@ -198,7 +227,7 @@ export default {
remapColNames(batchData, columns) { remapColNames(batchData, columns) {
return batchData.map(data => (columns || []).reduce((aggObj, col) => ({ return batchData.map(data => (columns || []).reduce((aggObj, col) => ({
...aggObj, ...aggObj,
[col.column_name]: data[col.refCn] [col.column_name]: data[col.ref_column_name]
}), {}) }), {})
) )
} }

1135
packages/nc-gui/components/templates/editor.vue

File diff suppressed because it is too large Load Diff

20
packages/nc-gui/pages/projects/index.vue

@ -135,6 +135,26 @@
<span class="caption">{{ $t("tooltip.extDB") }}</span> <span class="caption">{{ $t("tooltip.extDB") }}</span>
</v-tooltip> </v-tooltip>
</v-list-item> </v-list-item>
<v-divider />
<v-list-item
title
class="pt-2 nc-create-project-from-excel"
@click="onCreateProjectFromExcel()"
>
<v-list-item-icon class="mr-2">
<v-icon small class="">
mdi-file-excel-outline
</v-icon>
</v-list-item-icon>
<v-list-item-title>
<span
class="caption font-weight-regular"
v-html="
$t('activity.createProjectExtended.excel')
"
/>
</v-list-item-title>
</v-list-item>
</v-list> </v-list>
</v-menu> </v-menu>
</template> </template>

20
packages/noco-docs/content/en/setup-and-usages/dashboard.md

@ -30,7 +30,7 @@ Once you have logged into NocoDB, you should see `My Projects`.
To create a project, you can click `New Project`. To create a project, you can click `New Project`.
![image](https://user-images.githubusercontent.com/35857179/163135493-2afcc8c9-153b-4c82-9f41-397facd10b1a.png) ![image](https://user-images.githubusercontent.com/35857179/167252813-84876756-f6a1-488a-a185-cbb09f163c5b.png)
### Creating Empty Project ### Creating Empty Project
@ -96,20 +96,28 @@ Tip 3: You can click Edit Connection JSON and specify the schema you want to use
} }
``` ```
Click `Test Database Connection` to see if the connection can be established or not. NocoDB create's a new **empty database** with specified parameters, if the database doesn't exist. Click `Test Database Connection` to see if the connection can be established or not. NocoDB creates a new **empty database** with specified parameters if the database doesn't exist.
![image](https://user-images.githubusercontent.com/35857179/163136039-ad521d74-6996-4173-84ba-cfc55392c3b7.png) ![image](https://user-images.githubusercontent.com/35857179/163136039-ad521d74-6996-4173-84ba-cfc55392c3b7.png)
<!-- ### Creating Project from Excel ### Creating Project from Excel
Click `Create Project from Excel`, you can either upload / drag and drop Excel file (OR) specify Excel file URL. Click `Create Project from Excel`, you can either upload / drag and drop Excel file (OR) specify Excel file URL.
<alert> <alert type="success">
A local SQLite will be used. If your excel file contains multiple sheets, each sheet will be stored in a separate table. <br> Currently the data will be imported to NC_DB only. We'll support importing to existing projects with other database types in the future.
</alert> </alert>
<img src="https://user-images.githubusercontent.com/86527202/144373863-7ced9315-a70b-4746-9295-325e463dc110.png" width="60%"/> <img src="https://user-images.githubusercontent.com/86527202/144373863-7ced9315-a70b-4746-9295-325e463dc110.png" width="60%"/>
You can change Project Name, Table Name, Column Name or even Column Type as you want.
![image](https://user-images.githubusercontent.com/35857179/167252703-3a9be428-8737-4683-bc29-d3f9dbbfb712.png)
Click Import Excel to start importing process. The project and the table(s) will be created and the data will be imported to the corresponding table(s).
![image](https://user-images.githubusercontent.com/35857179/167253045-2e9890ca-4451-4b59-8eba-cb90ea5abdf1.png)
Supported file formats Supported file formats
- Xls - Xls
@ -117,4 +125,4 @@ Supported file formats
- Xlsm - Xlsm
- Ods - Ods
- Ots - Ots
-->

18
packages/nocodb-sdk/src/lib/Api.ts

@ -3073,6 +3073,24 @@ export class Api<
...params, ...params,
}), }),
/**
* @description Generic Axios Call
*
* @tags Utils
* @name AxiosRequestMake
* @request POST:/api/v1/db/meta/axiosRequestMake
* @response `200` `object` OK
*/
axiosRequestMake: (data: object, params: RequestParams = {}) =>
this.request<object, any>({
path: `/api/v1/db/meta/axiosRequestMake`,
method: 'POST',
body: data,
type: ContentType.Json,
format: 'json',
...params,
}),
/** /**
* No description * No description
* *

2
packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts

@ -1479,7 +1479,7 @@ class BaseModelSqlv2 {
const response = await this.dbDriver const response = await this.dbDriver
.batchInsert(this.model.table_name, insertDatas, 50) .batchInsert(this.model.table_name, insertDatas, 50)
.returning(this.model.primaryKey.column_name); .returning(this.model.primaryKey?.column_name);
// await this.afterInsertb(insertDatas, null); // await this.afterInsertb(insertDatas, null);

47
packages/nocodb/src/lib/noco/meta/api/utilApis.ts

@ -60,6 +60,52 @@ export async function releaseVersion(_req: Request, res: Response) {
res.json(result); res.json(result);
} }
export async function axiosRequestMake(req: Request, res: Response) {
const { apiMeta } = req.body;
if (apiMeta?.body) {
try {
apiMeta.body = JSON.parse(apiMeta.body);
} catch (e) {
console.log(e);
}
}
if (apiMeta?.auth) {
try {
apiMeta.auth = JSON.parse(apiMeta.auth);
} catch (e) {
console.log(e);
}
}
apiMeta.response = {};
const _req = {
params: apiMeta.parameters
? apiMeta.parameters.reduce((paramsObj, param) => {
if (param.name && param.enabled) {
paramsObj[param.name] = param.value;
}
return paramsObj;
}, {})
: {},
url: apiMeta.url,
method: apiMeta.method || 'GET',
data: apiMeta.body || {},
headers: apiMeta.headers
? apiMeta.headers.reduce((headersObj, header) => {
if (header.name && header.enabled) {
headersObj[header.name] = header.value;
}
return headersObj;
}, {})
: {},
responseType: apiMeta.responseType || 'json',
withCredentials: true
};
const data = await require('axios')(_req);
return res.json(data?.data);
}
export default router => { export default router => {
router.post( router.post(
'/api/v1/db/meta/connection/test', '/api/v1/db/meta/connection/test',
@ -67,4 +113,5 @@ export default router => {
); );
router.get('/api/v1/db/meta/nocodb/info', catchError(appInfo)); router.get('/api/v1/db/meta/nocodb/info', catchError(appInfo));
router.get('/api/v1/db/meta/nocodb/version', catchError(releaseVersion)); router.get('/api/v1/db/meta/nocodb/version', catchError(releaseVersion));
router.post('/api/v1/db/meta/axiosRequestMake', catchError(axiosRequestMake));
}; };

33
scripts/sdk/swagger.json

@ -4800,6 +4800,39 @@
"description": "" "description": ""
} }
}, },
"/api/v1/db/meta/axiosRequestMake": {
"parameters": [],
"post": {
"summary": "",
"operationId": "utils-axios-request-make",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {}
}
}
}
}
},
"description": "Generic Axios Call",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"tags": [
"Utils"
]
}
},
"/api/v1/db/meta/nocodb/version": { "/api/v1/db/meta/nocodb/version": {
"parameters": [], "parameters": [],
"get": { "get": {

Loading…
Cancel
Save