Browse Source

feat: Project export/import for external DB project

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/474/head
Pranav C 3 years ago
parent
commit
a901263b4d
  1. 77
      packages/nc-gui/components/createOrEditProject.vue
  2. 5
      packages/nc-gui/components/project/settings/xcMeta.vue
  3. 7
      packages/nocodb/package-lock.json
  4. 1
      packages/nocodb/package.json
  5. 38
      packages/nocodb/src/lib/noco/NcProjectBuilder.ts
  6. 23
      packages/nocodb/src/lib/noco/Noco.ts
  7. 21
      packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts
  8. 131
      packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts

77
packages/nc-gui/components/createOrEditProject.vue

@ -490,6 +490,7 @@
: 'password'
"
:label="$t('projects.ext_db.credentials.password')"
@dblclick="enableDbEdit++"
>
<template #append>
<v-icon
@ -516,6 +517,7 @@
<v-text-field
v-model="db.connection.database"
v-ge="['project', 'env-db-name']"
:disabled="edit && enableDbEdit < 2"
class="body-2 database-field"
:rules="form.requiredRule"
:label="$t('projects.ext_db.credentials.db_create_if_not_exists')"
@ -1046,6 +1048,7 @@ export default {
],
loaderMessage: '',
projectReloading: false,
enableDbEdit: 0,
authTypes: [
{ text: 'JWT', value: 'jwt' },
{ text: 'Master Key', value: 'masterKey' },
@ -1055,19 +1058,6 @@ export default {
projectTypes: [
{ text: 'REST APIs', value: 'rest', icon: 'mdi-code-json', iconColor: 'green' },
{ text: 'GRAPHQL APIs', value: 'graphql', icon: 'mdi-graphql', iconColor: 'pink' }
// {
// text: 'Automatic gRPC APIs on database',
// value: 'grpc',
// icon: require('@/assets/img/grpc-icon-color.png'),
// type: 'img'
// },
// {
// text: 'Automatic SQL Schema Migrations',
// value: 'migrations',
// icon: 'mdi-database-sync',
// iconColor: 'indigo'
// },
// {text: 'Simple Database Connection', value: 'dbConnection', icon: 'mdi-database', iconColor: 'primary'},
],
showPass: {},
@ -1317,9 +1307,9 @@ export default {
},
sslUse: this.$t('projects.ext_db.credentials.advanced.ssl.preferred'), // Preferred
ssl: {
key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key'), // Client Key
cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert'), // Client Cert
ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca') // Server CA
key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key.title'), // Client Key
cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert.title'), // Client Cert
ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca.title') // Server CA
},
databaseNames: {
MySQL: 'mysql2',
@ -1559,7 +1549,8 @@ export default {
options: JSON5.parse(this.smtpConfiguration.options),
from: this.smtpConfiguration.from
}
} catch (e) {}
} catch (e) {
}
}
xcConfig.meta = xcConfig.meta || {}
@ -1593,9 +1584,9 @@ export default {
Vue.set(db, 'ui', {
setup: 0,
ssl: {
key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key'), // Client Key
cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert'), // Client Cert
ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca') // Server CA
key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key.title'), // Client Key
cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert.title'), // Client Cert
ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca.title') // Server CA
},
sslUse: this.$t('projects.ext_db.credentials.advanced.ssl.preferred') // Preferred
})
@ -1641,8 +1632,12 @@ export default {
let i = 0
const toast = this.$toast.info(this.loaderMessages[0])
const interv = setInterval(() => {
if (this.edit) { return }
if (i < this.loaderMessages.length - 1) { i++ }
if (this.edit) {
return
}
if (i < this.loaderMessages.length - 1) {
i++
}
if (toast) {
if (!this.allSchemas) {
toast.text(this.loaderMessages[i])
@ -1849,7 +1844,7 @@ export default {
this.project.envs[env].db.length === 1 &&
this.project.envs[env].db[0].connection.user === 'postgres' &&
this.project.envs[env].db[0].connection.database ===
`${this.project.title}_${env}_${this.project.envs[env].length}`
`${this.project.title}_${env}_${this.project.envs[env].length}`
) {
this.handleSSL(db)
if (db.client === 'sqlite3') {
@ -1885,7 +1880,7 @@ export default {
for (const e in this.project.envs) {
if (e === env) {
// ignore
// ignore
} else {
console.log(this.project.envs[e])
@ -1932,7 +1927,9 @@ export default {
},
sendAdvancedConfig(connection) {
if (!connection.ssl) { return false }
if (!connection.ssl) {
return false
}
let sendAdvancedConfig = false
const sslOptions = Object.values(connection.ssl).filter(el => !!el)
console.log('sslOptions:', sslOptions)
@ -1963,7 +1960,8 @@ export default {
// }
}
},
getDatabaseForTestConnection(dbType) {},
getDatabaseForTestConnection(dbType) {
},
async testConnection(db, env, panelIndex) {
this.$store.commit('notification/MutToggleProgressBar', true)
try {
@ -2028,7 +2026,9 @@ export default {
return dbs.db.every(db => db.ui.setup === 1)
},
openFirstPanel() {
if (!this.edit) { this.panel = 0 }
if (!this.edit) {
this.panel = 0
}
},
onDatabaseTypeChanged(client, db1, index, env) {
for (const env in this.project.envs) {
@ -2072,7 +2072,9 @@ export default {
}
},
selectDatabaseClient(database, index = 0) {
if (this.client) { this.client[index] = database }
if (this.client) {
this.client[index] = database
}
},
setDBStatus(db, status) {
db.ui.setup = status
@ -2091,11 +2093,15 @@ export default {
Vue.set(this.project, 'envs', { ...this.project.envs })
}
},
fetch({ store, params }) {},
beforeCreated() {},
fetch({ store, params }) {
},
beforeCreated() {
},
watch: {
'project.title'(newValue, oldValue) {
if (!newValue) { return }
if (!newValue) {
return
}
if (!this.edit) {
// Vue.set(this.project, 'folder', slash(path.join(this.baseFolder, newValue)))
Vue.set(this.project, 'folder', [this.baseFolder, newValue].join('/'))
@ -2218,7 +2224,8 @@ export default {
// }
}
},
beforeMount() {},
beforeMount() {
},
mounted() {
this.$set(
this.project,
@ -2236,8 +2243,10 @@ export default {
input.focus()
})
},
beforeDestroy() {},
destroy() {},
beforeDestroy() {
},
destroy() {
},
validate({ params }) {
return true
},

5
packages/nc-gui/components/project/settings/xcMeta.vue

@ -283,11 +283,12 @@ export default {
this.$refs.importFile.value = ''
await this.$store.dispatch('sqlMgr/ActUpload', [
{
// dbAlias: 'db',
env: 'dev'
},
'xcMetaTablesImportZipToLocalFsAndDb',
{},
{
importsToCurrentProject: true
},
zipFile
])
this.$toast.success('Successfully imported metadata').goAway(3000)

7
packages/nocodb/package-lock.json generated

@ -1,6 +1,6 @@
{
"name": "nocodb",
"version": "0.11.21",
"version": "0.11.22",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -12827,6 +12827,11 @@
"nc-common": "0.0.6"
}
},
"ncp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
"integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M="
},
"needle": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/needle/-/needle-2.6.0.tgz",

1
packages/nocodb/package.json

@ -149,6 +149,7 @@
"nc-help": "^0.2.13",
"nc-lib-gui": "^0.2.32",
"nc-plugin": "^0.1.1",
"ncp": "^2.0.0",
"nodemailer": "^6.4.10",
"ora": "^4.0.4",
"os-locale": "^5.0.0",

38
packages/nocodb/src/lib/noco/NcProjectBuilder.ts

@ -21,7 +21,7 @@ export default class NcProjectBuilder {
public readonly description: string;
public readonly router: Router;
public readonly apiBuilders: Array<RestApiBuilder | GqlApiBuilder> = [];
public readonly config: any;
private _config: any;
protected startTime;
protected app: Noco;
@ -38,7 +38,7 @@ export default class NcProjectBuilder {
this.id = project.id;
this.title = project.title;
this.description = project.description;
this.config = {...this.appConfig, ...JSON.parse(project.config)};
this._config = {...this.appConfig, ...JSON.parse(project.config)};
this.router = Router();
}
}
@ -455,11 +455,7 @@ export default class NcProjectBuilder {
case 'xcMetaTablesImportLocalFsToDb':
case 'xcMetaTablesImportZipToLocalFsAndDb':
case 'projectRestart':
this.router.stack.splice(0, this.router.stack.length);
this.apiBuilders.splice(0, this.apiBuilders.length);
await this.app.ncMeta.projectStatusUpdate(this.title, 'stopped');
await this.init();
NcProjectBuilder.triggerGarbageCollect();
await this.reInit();
this.app.ncMeta.audit(this.id, null, 'nc_audit', {
op_type: 'PROJECT',
op_sub_type: 'RESTARTED',
@ -784,6 +780,34 @@ export default class NcProjectBuilder {
return this.config?.prefix;
}
public get config(): any {
return this._config;
}
public updateConfig(config: string) {
this._config = {...this.appConfig, ...JSON.parse(config)};
}
public async reInit() {
this.router.stack.splice(0, this.router.stack.length);
this.apiBuilders.splice(0, this.apiBuilders.length);
await this.app.ncMeta.projectStatusUpdate(this.title, 'stopped');
const dbs = this.config?.envs?.[this.appConfig.workingEnv]?.db
if (!dbs || !dbs.length) {
return;
}
for (const connectionConfig of dbs) {
NcConnectionMgr.delete({
dbAlias: connectionConfig?.mets?.dbAlias,
env: this.config.env,
projectId: this.id
})
}
NcProjectBuilder.triggerGarbageCollect();
await this.init();
}
}

23
packages/nocodb/src/lib/noco/Noco.ts

@ -294,21 +294,22 @@ export default class Noco {
const builder = new NcProjectBuilder(this, this.config, project);
this.projectBuilders.push(builder)
await builder.init(true);
} else {
const projectBuilder = this.projectBuilders.find(pb => pb.id == data.req?.project_id);
return projectBuilder?.handleRunTimeChanges(data);
}
}
break;
case 'projectUpdateByWeb':
this.config.toolDir = this.config.toolDir || process.cwd();
this.config.workingEnv = this.env;
this.ncMeta.setConfig(this.config);
this.metaMgr.setConfig(this.config);
this.router.stack.splice(0, this.router.stack.length);
this.ncToolApi.destroy();
this.ncToolApi.reInitialize(this.config);
this.initWebSocket();
await this.init({});
console.log(`Project created: ${data.req.args.tn}`)
case 'projectUpdateByWeb': {
const projectId = data.req?.project_id;
const project = await this.ncMeta.projectGetById(data?.req?.project_id)
const projectBuilder = this.projectBuilders.find(pb => pb.id === projectId);
projectBuilder.updateConfig(project.config)
await projectBuilder.reInit()
console.log(`Project updated: ${projectId}`)
}
break;
case 'projectChangeEnv':

21
packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts

@ -21,6 +21,27 @@ export default class NcConnectionMgr {
this.metaKnex = ncMeta;
}
public static delete({
dbAlias = 'db',
env = 'dev',
projectId
}: {
dbAlias: string,
env: string,
projectId: string
}) {
// todo: ignore meta projects
if (this.connectionRefs?.[projectId]?.[env]?.[dbAlias]) {
try {
const conn = this.connectionRefs[projectId][env][dbAlias];
conn.destroy();
delete this.connectionRefs[projectId][env][dbAlias];
} catch (e) {
console.log(e);
}
}
}
public static get({
dbAlias = 'db',
env = 'dev',

131
packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts

@ -15,6 +15,7 @@ import {
} from 'nc-help'
import slash from 'slash';
import {v4 as uuidv4} from 'uuid';
import {ncp} from 'ncp';
import IEmailAdapter from "../../../interface/IEmailAdapter";
import IStorageAdapter from "../../../interface/IStorageAdapter";
@ -37,6 +38,7 @@ import {RestApiBuilder} from "../rest/RestApiBuilder";
import RestAuthCtrl from "../rest/RestAuthCtrlEE";
import {packageVersion} from 'nc-help';
import NcMetaIO, {META_TABLES} from "./NcMetaIO";
import {promisify} from "util";
const XC_PLUGIN_DET = 'XC_PLUGIN_DET';
@ -210,19 +212,19 @@ export default class NcMetaMgr {
let projectHasAdmin = false;
projectHasAdmin = !!(await knex('xc_users').first())
const result = {
authType: 'jwt',
projectHasAdmin,
firstUser: !projectHasAdmin,
projectHasDb,
type: this.config.type,
env: this.config.workingEnv,
googleAuthEnabled: !!(process.env.NC_GOOGLE_CLIENT_ID && process.env.NC_GOOGLE_CLIENT_SECRET),
githubAuthEnabled: !!(process.env.NC_GITHUB_CLIENT_ID && process.env.NC_GITHUB_CLIENT_SECRET),
oneClick: !!process.env.NC_ONE_CLICK,
connectToExternalDB: !process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED,
version: packageVersion
};
const result = {
authType: 'jwt',
projectHasAdmin,
firstUser: !projectHasAdmin,
projectHasDb,
type: this.config.type,
env: this.config.workingEnv,
googleAuthEnabled: !!(process.env.NC_GOOGLE_CLIENT_ID && process.env.NC_GOOGLE_CLIENT_SECRET),
githubAuthEnabled: !!(process.env.NC_GITHUB_CLIENT_ID && process.env.NC_GITHUB_CLIENT_SECRET),
oneClick: !!process.env.NC_ONE_CLICK,
connectToExternalDB: !process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED,
version: packageVersion
};
return res.json(result)
}
if (this.config.auth.masterKey) {
@ -303,7 +305,6 @@ const result = {
}
return res.json(result);
}
@ -352,7 +353,11 @@ const result = {
const data = JSON.parse(fs.readFileSync(path.join(metaFolder, `${tn}.json`), 'utf8'));
for (const row of data) {
delete row.id;
await this.xcMeta.metaInsert(projectId, dbAlias, tn, row)
await this.xcMeta.metaInsert(projectId, dbAlias, tn, {
...row,
db_alias: dbAlias,
project_id: projectId
})
}
}
}
@ -376,6 +381,98 @@ const result = {
// NOTE: xc-meta
// Extract and import metadata and config from zip file
public async xcMetaTablesImportZipToLocalFsAndDb(args, file, req) {
try {
await this.xcMetaTablesReset(args);
let projectConfigPath;
// let storeFilePath;
await extract(file.path, {
dir: path.join(this.config.toolDir, 'uploads'),
onEntry(entry, _zipfile) {
// extract xc_project.json file path
if (entry.fileName?.endsWith('nc_project.json')) {
projectConfigPath = entry.fileName;
}
}
});
// delete temporary upload file
fs.unlinkSync(file.path);
let projectId = this.getProjectId(args)
if (!projectConfigPath) {
throw new Error('Missing project config file')
}
const projectDetailsJSON: any = fs.readFileSync(path.join(this.config.toolDir, 'uploads', projectConfigPath), 'utf8');
const projectDetails = projectDetailsJSON && JSON.parse(projectDetailsJSON);
if (args.args.importsToCurrentProject) {
await promisify(ncp)(path.join(this.config.toolDir, 'uploads', 'nc', projectDetails.id), path.join(this.config.toolDir, 'nc', projectId), {clobber:true})
} else {
// decrypt with old key and encrypt again with latest key
const projectConfig = JSON.parse(CryptoJS.AES.decrypt(projectDetails.config, projectDetails.key).toString(CryptoJS.enc.Utf8))
// delete projectDetails.key;
projectDetails.config = projectConfig;
// create new project and import
const project = await this.xcMeta.projectCreate(projectDetails.title, projectConfig, projectDetails.description);
projectId = project.id;
// move files to newly created project meta folder
await promisify(ncp)(path.join(this.config.toolDir, 'uploads', 'nc', projectDetails.id), path.join(this.config.toolDir, 'nc', projectId))
await this.xcMeta.projectAddUser(projectId, req?.session?.passport?.user?.id, 'owner,creator');
await this.projectMgr.getSqlMgr({
...projectConfig,
metaDb: this.xcMeta?.knex
}).projectOpenByWeb(projectConfig);
this.projectConfigs[projectId] = projectConfig;
args.freshImport = true;
}
// const importProjectId = projectConfig?.id;
//
// // check project already exist
// if (await this.xcMeta.projectGetById(importProjectId)) {
// // todo:
// throw new Error(`Project with id '${importProjectId}' already exist, it's not allowed at the moment`)
// } else {
// // create the project if not found
// await this.xcMeta.knex('nc_projects').insert(projectConfig);
// projectConfig = JSON.parse((await this.xcMeta.projectGetById(importProjectId))?.config);
//
// // duplicated code from project create - see projectCreateByWeb
// await this.xcMeta.projectAddUser(importProjectId, req?.session?.passport?.user?.id, 'owner,creator');
// await this.projectMgr.getSqlMgr({
// ...projectConfig,
// metaDb: this.xcMeta?.knex
// }).projectOpenByWeb(projectConfig);
// this.projectConfigs[importProjectId] = projectConfig;
//
// args.freshImport = true;
// }
// args.project_id = importProjectId;
// }
args.project_id = projectId
await this.xcMetaTablesImportLocalFsToDb(args, req);
this.xcMeta.audit(projectId, null, 'nc_audit', {
op_type: 'META',
op_sub_type: 'IMPORT_FROM_ZIP',
user: req.user.email,
description: `imported ${projectId} from zip file uploaded `, ip: req.clientIp
})
} catch (e) {
throw e;
}
}
// NOTE: xc-meta
// Extract and import metadata and config from zip file
public async xcMetaTablesImportZipToLocalFsAndDbV1(args, file, req) {
try {
await this.xcMetaTablesReset(args);
let projectConfigPath;
@ -2401,7 +2498,7 @@ const result = {
childColumn: `${parentMeta.tn}_p_id`,
parentTable: parentMeta.tn,
parentColumn: parentPK.cn,
foreignKeyName:`${parentMeta.tn.slice(0,3)}_${childMeta.tn.slice(0,3)}_${nanoid(6)}_p_fk`,
foreignKeyName: `${parentMeta.tn.slice(0, 3)}_${childMeta.tn.slice(0, 3)}_${nanoid(6)}_p_fk`,
type: 'real'
};
const rel2Args = {
@ -2410,7 +2507,7 @@ const result = {
childColumn: `${childMeta.tn}_c_id`,
parentTable: childMeta.tn,
parentColumn: childPK.cn,
foreignKeyName:`${parentMeta.tn.slice(0,3)}_${childMeta.tn.slice(0,3)}_${nanoid(6)}_c_fk`,
foreignKeyName: `${parentMeta.tn.slice(0, 3)}_${childMeta.tn.slice(0, 3)}_${nanoid(6)}_c_fk`,
type: 'real'
};
if (args.args.type === 'real') {

Loading…
Cancel
Save