Browse Source

Merge pull request #1931 from nocodb/fix/excel-import

fix: excel import
pull/1998/head
աɨռɢӄաօռɢ 2 years ago committed by GitHub
parent
commit
89048d61ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      packages/nc-gui/components/import/excelImport.vue
  2. 6
      packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js
  3. 12
      packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js
  4. 199
      packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue
  5. 1135
      packages/nc-gui/components/templates/editor.vue
  6. 20
      packages/nc-gui/pages/projects/index.vue
  7. 22
      packages/noco-docs/content/en/setup-and-usages/dashboard.md
  8. 18
      packages/nocodb-sdk/src/lib/Api.ts
  9. 2
      packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
  10. 47
      packages/nocodb/src/lib/noco/meta/api/utilApis.ts
  11. 33
      scripts/sdk/swagger.json

4
packages/nc-gui/components/import/excelImport.vue

@ -66,7 +66,7 @@
dense
:rules="[v => !!v || $t('general.required') ]"
/>
<v-btn class="ml-3" color="primary" @click="loadUrl">
<v-btn v-t="['c:project:create:excel:load-url']" class="ml-3" color="primary" @click="loadUrl">
<!--Load-->
{{ $t('general.load') }}
</v-btn>
@ -266,7 +266,7 @@ export default {
templateGenerator = new ExcelTemplateAdapter(name, val, this.parserConfig)
break
case 'url':
templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig)
templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig, this.$api)
break
}
await templateGenerator.init()

6
packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@ -42,7 +42,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
}
tableNamePrefixRef[tn] = 0
const table = { tn, refTn: tn, columns: [] }
const table = { table_name: tn, ref_table_name: tn, columns: [] }
this.data[tn] = []
const ws = this.wb.Sheets[sheet]
const range = XLSX.utils.decode_range(ws['!ref'])
@ -79,8 +79,8 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
columnNamePrefixRef[cn] = 0
const column = {
cn,
refCn: cn
column_name: cn,
ref_column_name: cn
}
table.columns.push(column)

12
packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js

@ -1,20 +1,22 @@
import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter'
export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter {
constructor(url, $store, parserConfig) {
constructor(url, $store, parserConfig, $api) {
const name = url.split('/').pop()
super(name, null, parserConfig)
this.url = url
this.$api = $api
this.$store = $store
}
async init() {
const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'handleAxiosCall',
[{
const data = await this.$api.utils.axiosRequestMake({
apiMeta: {
url: this.url,
responseType: 'arraybuffer'
}]])
this.excelData = res.data
}
})
this.excelData = data.data
await super.init()
}
}

199
packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue

@ -13,6 +13,7 @@
</template>
<script>
import { SqlUiFactory } from 'nocodb-sdk'
import colors from '~/mixins/colors'
export default {
@ -43,7 +44,9 @@ export default {
},
data() {
return {
localTemplateData: null,
projectCreation: false,
tableCreation: false,
loaderMessagesIndex: 0,
loaderMessages: [
'Setting up new database configs',
@ -69,6 +72,18 @@ export default {
]
}
},
watch: {
templateData: {
deep: true,
handler(data) {
this.localTemplateData = JSON.parse(JSON.stringify(data))
}
}
},
created() {
this.localTemplateData = JSON.parse(JSON.stringify(this.templateData))
},
methods: {
async useTemplate(projectType) {
if (!this.valid) {
@ -76,8 +91,7 @@ export default {
}
// this.$emit('useTemplate', type)
this.projectCreation = true
// this.projectCreation = true
let interv
try {
interv = setInterval(() => {
@ -85,111 +99,126 @@ export default {
this.$store.commit('loader/MutMessage', this.loaderMessages[this.loaderMessagesIndex])
}, 1000)
let projectId, prefix
let project
// Not available now
if (this.importToProject) {
this.$store.commit('loader/MutMessage', 'Importing excel template')
const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [{
// todo: extract based on active
dbAlias: 'db', // this.nodes.dbAlias,
env: '_noco'
}, 'xcModelsCreateFromTemplate', {
template: this.templateData
}])
if (res && res.tables && res.tables.length) {
this.$toast.success(`Imported ${res.tables.length} tables successfully`).goAway(3000)
} else {
this.$toast.success('Template imported successfully').goAway(3000)
// this.$store.commit('loader/MutMessage', 'Importing excel template')
// const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [{
// // todo: extract based on active
// dbAlias: 'db', // this.nodes.dbAlias,
// env: '_noco'
// }, 'xcModelsCreateFromTemplate', {
// template: this.templateData
// }])
// if (res && res.tables && res.tables.length) {
// this.$toast.success(`Imported ${res.tables.length} tables successfully`).goAway(3000)
// } else {
// this.$toast.success('Template imported successfully').goAway(3000)
// }
// projectId = this.$route.params.project_id
// prefix = this.$store.getters['project/GtrProjectPrefix']
} else {
// Create an empty project
try {
this.$e("a:project:create:excel");
project = await this.$api.project.create({
title: this.templateData.title,
external: false
})
this.projectCreation = true
} catch (e) {
this.projectCreation = false
this.$toast
.error(await this._extractSdkResponseErrorMsg(e))
.goAway(3000)
} finally {
clearInterval(interv)
}
projectId = this.$route.params.project_id
prefix = this.$store.getters['project/GtrProjectPrefix']
} else {
const result = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'projectCreateByWebWithXCDB', {
title: this.templateData.title,
projectType,
template: this.templateData,
excelImport: this.excelImport
}])
projectId = result.id
prefix = result.prefix
await this.$store.dispatch('project/ActLoadProjectInfo')
if (!this.projectCreation) {
// failed to create project
return
}
// Create tables
try {
for (const t of this.localTemplateData.tables) {
// enrich system fields if not provided
// e.g. id, created_at, updated_at
const systemColumns = SqlUiFactory
.create({ client: 'sqlite3' })
.getNewTableColumns()
.filter(c => c.column_name != 'title')
const table = await this.$api.dbTable.create(project.id, {
table_name: t.table_name,
title: '',
columns: [...t.columns, ...systemColumns]
})
t.table_title = table.title
}
this.tableCreation = true
} catch (e) {
this.$toast
.error(await this._extractSdkResponseErrorMsg(e))
.goAway(3000)
this.tableCreation = false
} finally {
clearInterval(interv)
}
}
clearInterval(interv)
if (!this.tableCreation) {
// failed to create table
return
}
// Bulk import data
if (this.importData) {
this.$store.commit('loader/MutMessage', 'Importing excel data to project')
await this.importDataToProject({ projectId, projectType, prefix })
await this.importDataToProject(this.templateData.title)
}
this.$store.commit('loader/MutMessage', null)
this.projectReloading = false
if (!this.importToProject) {
await this.$router.push({
path: `/nc/${projectId}`,
query: {
new: 1
}
})
}
this.$router.push({
path: `/nc/${project.id}`,
query: {
new: 1
}
})
this.$emit('success')
} catch (e) {
console.log(e)
this.$toast.error(e.message).goAway(3000)
this.$store.commit('loader/MutMessage', null)
} finally {
clearInterval(interv)
this.$store.commit('loader/MutMessage', null)
this.projectCreation = false
this.tableCreation = false
}
this.projectCreation = false
},
async importDataToProject({ projectId, projectType, prefix = '' }) {
// this.$store.commit('project/MutProjectId', projectId)
this.$ncApis.setProjectId(projectId)
async importDataToProject(projectName) {
let total = 0
let progress = 0
/* await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => {
await this.$store.dispatch('meta/ActLoadMeta', {
tn: `${prefix}${table}`, project_id: projectId
})
// todo: get table name properly
const api = this.$ncApis.get({
table: `${prefix}${table}`,
type: projectType
})
total += data.length
for (let i = 0; i < data.length; i += 500) {
this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`)
this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
const batchData = data.slice(i, i + 500)
await api.insertBulk(batchData)
progress += batchData.length
}
this.$store.commit('loader/MutClear')
})(v))) */
await Promise.all(this.templateData.tables.map(v => (async(tableMeta) => {
const table = tableMeta.table_name
const data = this.importData[tableMeta.refTn]
await this.$store.dispatch('meta/ActLoadMeta', {
tn: `${prefix}${table}`, project_id: projectId
})
// todo: get table name properly
const api = this.$ncApis.get({
table: `${prefix}${table}`,
type: projectType
})
await Promise.all(this.localTemplateData.tables.map(v => (async(tableMeta) => {
const tableName = tableMeta.table_title
const data = this.importData[tableMeta.ref_table_name]
total += data.length
for (let i = 0; i < data.length; i += 500) {
this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`)
this.$store.commit('loader/MutMessage', `Importing data to ${projectName}: ${progress}/${total} records`)
this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
const batchData = this.remapColNames(data.slice(i, i + 500), tableMeta.columns)
await api.insertBulk(batchData)
await this.$api.dbTableRow.bulkCreate(
'noco',
projectName,
tableName,
batchData
)
progress += batchData.length
}
this.$store.commit('loader/MutClear')
@ -198,7 +227,7 @@ export default {
remapColNames(batchData, columns) {
return batchData.map(data => (columns || []).reduce((aggObj, col) => ({
...aggObj,
[col.column_name]: data[col.refCn]
[col.column_name]: data[col.ref_column_name]
}), {})
)
}

1135
packages/nc-gui/components/templates/editor.vue

File diff suppressed because it is too large Load Diff

20
packages/nc-gui/pages/projects/index.vue

@ -135,6 +135,26 @@
<span class="caption">{{ $t("tooltip.extDB") }}</span>
</v-tooltip>
</v-list-item>
<v-divider />
<v-list-item
title
class="pt-2 nc-create-project-from-excel"
@click="onCreateProjectFromExcel()"
>
<v-list-item-icon class="mr-2">
<v-icon small class="">
mdi-file-excel-outline
</v-icon>
</v-list-item-icon>
<v-list-item-title>
<span
class="caption font-weight-regular"
v-html="
$t('activity.createProjectExtended.excel')
"
/>
</v-list-item-title>
</v-list-item>
</v-list>
</v-menu>
</template>

22
packages/noco-docs/content/en/setup-and-usages/dashboard.md

@ -30,7 +30,7 @@ Once you have logged into NocoDB, you should see `My Projects`.
To create a project, you can click `New Project`.
![image](https://user-images.githubusercontent.com/35857179/163135493-2afcc8c9-153b-4c82-9f41-397facd10b1a.png)
![image](https://user-images.githubusercontent.com/35857179/167252813-84876756-f6a1-488a-a185-cbb09f163c5b.png)
### Creating Empty Project
@ -96,20 +96,28 @@ Tip 3: You can click Edit Connection JSON and specify the schema you want to use
}
```
Click `Test Database Connection` to see if the connection can be established or not. NocoDB create's a new **empty database** with specified parameters, if the database doesn't exist.
Click `Test Database Connection` to see if the connection can be established or not. NocoDB creates a new **empty database** with specified parameters if the database doesn't exist.
![image](https://user-images.githubusercontent.com/35857179/163136039-ad521d74-6996-4173-84ba-cfc55392c3b7.png)
<!-- ### Creating Project from Excel
### Creating Project from Excel
Click `Create Project from Excel`, you can either upload / drag and drop Excel file (OR) specify Excel file URL.
Click `Create Project from Excel`, you can either upload / drag and drop Excel file (OR) specify Excel file URL.
<alert>
A local SQLite will be used.
<alert type="success">
If your excel file contains multiple sheets, each sheet will be stored in a separate table. <br> Currently the data will be imported to NC_DB only. We'll support importing to existing projects with other database types in the future.
</alert>
<img src="https://user-images.githubusercontent.com/86527202/144373863-7ced9315-a70b-4746-9295-325e463dc110.png" width="60%"/>
You can change Project Name, Table Name, Column Name or even Column Type as you want.
![image](https://user-images.githubusercontent.com/35857179/167252703-3a9be428-8737-4683-bc29-d3f9dbbfb712.png)
Click Import Excel to start importing process. The project and the table(s) will be created and the data will be imported to the corresponding table(s).
![image](https://user-images.githubusercontent.com/35857179/167253045-2e9890ca-4451-4b59-8eba-cb90ea5abdf1.png)
Supported file formats
- Xls
@ -117,4 +125,4 @@ Supported file formats
- Xlsm
- Ods
- Ots
-->

18
packages/nocodb-sdk/src/lib/Api.ts

@ -3073,6 +3073,24 @@ export class Api<
...params,
}),
/**
* @description Generic Axios Call
*
* @tags Utils
* @name AxiosRequestMake
* @request POST:/api/v1/db/meta/axiosRequestMake
* @response `200` `object` OK
*/
axiosRequestMake: (data: object, params: RequestParams = {}) =>
this.request<object, any>({
path: `/api/v1/db/meta/axiosRequestMake`,
method: 'POST',
body: data,
type: ContentType.Json,
format: 'json',
...params,
}),
/**
* No description
*

2
packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts

@ -1479,7 +1479,7 @@ class BaseModelSqlv2 {
const response = await this.dbDriver
.batchInsert(this.model.table_name, insertDatas, 50)
.returning(this.model.primaryKey.column_name);
.returning(this.model.primaryKey?.column_name);
// await this.afterInsertb(insertDatas, null);

47
packages/nocodb/src/lib/noco/meta/api/utilApis.ts

@ -60,6 +60,52 @@ export async function releaseVersion(_req: Request, res: Response) {
res.json(result);
}
export async function axiosRequestMake(req: Request, res: Response) {
const { apiMeta } = req.body;
if (apiMeta?.body) {
try {
apiMeta.body = JSON.parse(apiMeta.body);
} catch (e) {
console.log(e);
}
}
if (apiMeta?.auth) {
try {
apiMeta.auth = JSON.parse(apiMeta.auth);
} catch (e) {
console.log(e);
}
}
apiMeta.response = {};
const _req = {
params: apiMeta.parameters
? apiMeta.parameters.reduce((paramsObj, param) => {
if (param.name && param.enabled) {
paramsObj[param.name] = param.value;
}
return paramsObj;
}, {})
: {},
url: apiMeta.url,
method: apiMeta.method || 'GET',
data: apiMeta.body || {},
headers: apiMeta.headers
? apiMeta.headers.reduce((headersObj, header) => {
if (header.name && header.enabled) {
headersObj[header.name] = header.value;
}
return headersObj;
}, {})
: {},
responseType: apiMeta.responseType || 'json',
withCredentials: true
};
const data = await require('axios')(_req);
return res.json(data?.data);
}
export default router => {
router.post(
'/api/v1/db/meta/connection/test',
@ -67,4 +113,5 @@ export default router => {
);
router.get('/api/v1/db/meta/nocodb/info', catchError(appInfo));
router.get('/api/v1/db/meta/nocodb/version', catchError(releaseVersion));
router.post('/api/v1/db/meta/axiosRequestMake', catchError(axiosRequestMake));
};

33
scripts/sdk/swagger.json

@ -4800,6 +4800,39 @@
"description": ""
}
},
"/api/v1/db/meta/axiosRequestMake": {
"parameters": [],
"post": {
"summary": "",
"operationId": "utils-axios-request-make",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {}
}
}
}
}
},
"description": "Generic Axios Call",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"tags": [
"Utils"
]
}
},
"/api/v1/db/meta/nocodb/version": {
"parameters": [],
"get": {

Loading…
Cancel
Save