feat: Project export/import for external DB project

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nc-gui/components/createOrEditProject.vue

@@ -490,6 +490,7 @@
                                                 : 'password'
                                             "
                                             :label="$t('projects.ext_db.credentials.password')"
+                                            @dblclick="enableDbEdit++"
                                           >
                                             <template #append>
                                               <v-icon
@@ -516,6 +517,7 @@
                                           <v-text-field
                                             v-model="db.connection.database"
                                             v-ge="['project', 'env-db-name']"
+                                            :disabled="edit && enableDbEdit < 2"
                                             class="body-2 database-field"
                                             :rules="form.requiredRule"
                                             :label="$t('projects.ext_db.credentials.db_create_if_not_exists')"
@@ -1046,6 +1048,7 @@ export default {
       ],
       loaderMessage: '',
       projectReloading: false,
+      enableDbEdit: 0,
       authTypes: [
         { text: 'JWT', value: 'jwt' },
         { text: 'Master Key', value: 'masterKey' },
@@ -1055,19 +1058,6 @@ export default {
       projectTypes: [
         { text: 'REST APIs', value: 'rest', icon: 'mdi-code-json', iconColor: 'green' },
         { text: 'GRAPHQL APIs', value: 'graphql', icon: 'mdi-graphql', iconColor: 'pink' }
-        // {
-        //   text: 'Automatic gRPC APIs on database',
-        //   value: 'grpc',
-        //   icon: require('@/assets/img/grpc-icon-color.png'),
-        //   type: 'img'
-        // },
-        // {
-        //   text: 'Automatic SQL Schema Migrations',
-        //   value: 'migrations',
-        //   icon: 'mdi-database-sync',
-        //   iconColor: 'indigo'
-        // },
-        // {text: 'Simple Database Connection', value: 'dbConnection', icon: 'mdi-database', iconColor: 'primary'},
       ],
 
       showPass: {},
@@ -1317,9 +1307,9 @@ export default {
       },
       sslUse: this.$t('projects.ext_db.credentials.advanced.ssl.preferred'), // Preferred
       ssl: {
-        key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key'), // Client Key
-        cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert'), // Client Cert
-        ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca') // Server CA
+        key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key.title'), // Client Key
+        cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert.title'), // Client Cert
+        ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca.title') // Server CA
       },
       databaseNames: {
         MySQL: 'mysql2',
@@ -1559,7 +1549,8 @@ export default {
             options: JSON5.parse(this.smtpConfiguration.options),
             from: this.smtpConfiguration.from
           }
-        } catch (e) {}
+        } catch (e) {
+        }
       }
 
       xcConfig.meta = xcConfig.meta || {}
@@ -1593,9 +1584,9 @@ export default {
           Vue.set(db, 'ui', {
             setup: 0,
             ssl: {
-              key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key'), // Client Key
-              cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert'), // Client Cert
-              ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca') // Server CA
+              key: this.$t('projects.ext_db.credentials.advanced.ssl.client_key.title'), // Client Key
+              cert: this.$t('projects.ext_db.credentials.advanced.ssl.client_cert.title'), // Client Cert
+              ca: this.$t('projects.ext_db.credentials.advanced.ssl.server_ca.title') // Server CA
             },
             sslUse: this.$t('projects.ext_db.credentials.advanced.ssl.preferred') // Preferred
           })
@@ -1641,8 +1632,12 @@ export default {
       let i = 0
       const toast = this.$toast.info(this.loaderMessages[0])
       const interv = setInterval(() => {
-        if (this.edit) { return }
-        if (i < this.loaderMessages.length - 1) { i++ }
+        if (this.edit) {
+          return
+        }
+        if (i < this.loaderMessages.length - 1) {
+          i++
+        }
         if (toast) {
           if (!this.allSchemas) {
             toast.text(this.loaderMessages[i])
@@ -1932,7 +1927,9 @@ export default {
     },
 
     sendAdvancedConfig(connection) {
-      if (!connection.ssl) { return false }
+      if (!connection.ssl) {
+        return false
+      }
       let sendAdvancedConfig = false
       const sslOptions = Object.values(connection.ssl).filter(el => !!el)
       console.log('sslOptions:', sslOptions)
@@ -1963,7 +1960,8 @@ export default {
         // }
       }
     },
-    getDatabaseForTestConnection(dbType) {},
+    getDatabaseForTestConnection(dbType) {
+    },
     async testConnection(db, env, panelIndex) {
       this.$store.commit('notification/MutToggleProgressBar', true)
       try {
@@ -2028,7 +2026,9 @@ export default {
       return dbs.db.every(db => db.ui.setup === 1)
     },
     openFirstPanel() {
-      if (!this.edit) { this.panel = 0 }
+      if (!this.edit) {
+        this.panel = 0
+      }
     },
     onDatabaseTypeChanged(client, db1, index, env) {
       for (const env in this.project.envs) {
@@ -2072,7 +2072,9 @@ export default {
       }
     },
     selectDatabaseClient(database, index = 0) {
-      if (this.client) { this.client[index] = database }
+      if (this.client) {
+        this.client[index] = database
+      }
     },
     setDBStatus(db, status) {
       db.ui.setup = status
@@ -2091,11 +2093,15 @@ export default {
       Vue.set(this.project, 'envs', { ...this.project.envs })
     }
   },
-  fetch({ store, params }) {},
-  beforeCreated() {},
+  fetch({ store, params }) {
+  },
+  beforeCreated() {
+  },
   watch: {
     'project.title'(newValue, oldValue) {
-      if (!newValue) { return }
+      if (!newValue) {
+        return
+      }
       if (!this.edit) {
         // Vue.set(this.project, 'folder', slash(path.join(this.baseFolder, newValue)))
         Vue.set(this.project, 'folder', [this.baseFolder, newValue].join('/'))
@@ -2218,7 +2224,8 @@ export default {
       // }
     }
   },
-  beforeMount() {},
+  beforeMount() {
+  },
   mounted() {
     this.$set(
       this.project,
@@ -2236,8 +2243,10 @@ export default {
       input.focus()
     })
   },
-  beforeDestroy() {},
-  destroy() {},
+  beforeDestroy() {
+  },
+  destroy() {
+  },
   validate({ params }) {
     return true
   },

packages/nc-gui/components/project/settings/xcMeta.vue

@@ -283,11 +283,12 @@ export default {
           this.$refs.importFile.value = ''
           await this.$store.dispatch('sqlMgr/ActUpload', [
             {
-              // dbAlias: 'db',
               env: 'dev'
             },
             'xcMetaTablesImportZipToLocalFsAndDb',
-            {},
+            {
+              importsToCurrentProject: true
+            },
             zipFile
           ])
           this.$toast.success('Successfully imported metadata').goAway(3000)

packages/nocodb/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "nocodb",
-  "version": "0.11.21",
+  "version": "0.11.22",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -12827,6 +12827,11 @@
         "nc-common": "0.0.6"
       }
     },
+    "ncp": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
+      "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M="
+    },
     "needle": {
       "version": "2.6.0",
       "resolved": "https://registry.npmjs.org/needle/-/needle-2.6.0.tgz",

packages/nocodb/package.json

@@ -149,6 +149,7 @@
     "nc-help": "^0.2.13",
     "nc-lib-gui": "^0.2.32",
     "nc-plugin": "^0.1.1",
+    "ncp": "^2.0.0",
     "nodemailer": "^6.4.10",
     "ora": "^4.0.4",
     "os-locale": "^5.0.0",

packages/nocodb/src/lib/noco/NcProjectBuilder.ts

@@ -21,7 +21,7 @@ export default class NcProjectBuilder {
   public readonly description: string;
   public readonly router: Router;
   public readonly apiBuilders: Array<RestApiBuilder | GqlApiBuilder> = [];
-  public readonly config: any;
+  private _config: any;
 
   protected startTime;
   protected app: Noco;
@@ -38,7 +38,7 @@ export default class NcProjectBuilder {
       this.id = project.id;
       this.title = project.title;
       this.description = project.description;
-      this.config = {...this.appConfig, ...JSON.parse(project.config)};
+      this._config = {...this.appConfig, ...JSON.parse(project.config)};
       this.router = Router();
     }
   }
@@ -455,11 +455,7 @@ export default class NcProjectBuilder {
       case 'xcMetaTablesImportLocalFsToDb':
       case 'xcMetaTablesImportZipToLocalFsAndDb':
       case 'projectRestart':
-        this.router.stack.splice(0, this.router.stack.length);
-        this.apiBuilders.splice(0, this.apiBuilders.length);
-        await this.app.ncMeta.projectStatusUpdate(this.title, 'stopped');
-        await this.init();
-        NcProjectBuilder.triggerGarbageCollect();
+        await this.reInit();
         this.app.ncMeta.audit(this.id, null, 'nc_audit', {
           op_type: 'PROJECT',
           op_sub_type: 'RESTARTED',
@@ -784,6 +780,34 @@ export default class NcProjectBuilder {
     return this.config?.prefix;
   }
 
+  public get config(): any {
+    return this._config;
+  }
+
+  public updateConfig(config: string) {
+    this._config = {...this.appConfig, ...JSON.parse(config)};
+  }
+
+  public async reInit() {
+    this.router.stack.splice(0, this.router.stack.length);
+    this.apiBuilders.splice(0, this.apiBuilders.length);
+    await this.app.ncMeta.projectStatusUpdate(this.title, 'stopped');
+    const dbs = this.config?.envs?.[this.appConfig.workingEnv]?.db
+
+    if (!dbs || !dbs.length) {
+      return;
+    }
+
+    for (const connectionConfig of dbs) {
+      NcConnectionMgr.delete({
+        dbAlias: connectionConfig?.mets?.dbAlias,
+        env: this.config.env,
+        projectId: this.id
+      })
+    }
+    NcProjectBuilder.triggerGarbageCollect();
+    await this.init();
+  }
 
 }
 

packages/nocodb/src/lib/noco/Noco.ts

@@ -294,21 +294,22 @@ export default class Noco {
             const builder = new NcProjectBuilder(this, this.config, project);
             this.projectBuilders.push(builder)
             await builder.init(true);
+          } else {
+            const projectBuilder = this.projectBuilders.find(pb => pb.id == data.req?.project_id);
+            return projectBuilder?.handleRunTimeChanges(data);
           }
         }
           break;
 
-        case 'projectUpdateByWeb':
-          this.config.toolDir = this.config.toolDir || process.cwd();
-          this.config.workingEnv = this.env;
-          this.ncMeta.setConfig(this.config);
-          this.metaMgr.setConfig(this.config);
-          this.router.stack.splice(0, this.router.stack.length);
-          this.ncToolApi.destroy();
-          this.ncToolApi.reInitialize(this.config);
-          this.initWebSocket();
-          await this.init({});
-          console.log(`Project created: ${data.req.args.tn}`)
+        case 'projectUpdateByWeb': {
+          const projectId = data.req?.project_id;
+          const project = await this.ncMeta.projectGetById(data?.req?.project_id)
+          const projectBuilder = this.projectBuilders.find(pb => pb.id === projectId);
+
+          projectBuilder.updateConfig(project.config)
+          await projectBuilder.reInit()
+          console.log(`Project updated: ${projectId}`)
+        }
           break;
 
         case 'projectChangeEnv':

packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts

@@ -21,6 +21,27 @@ export default class NcConnectionMgr {
     this.metaKnex = ncMeta;
   }
 
+  public static delete({
+                         dbAlias = 'db',
+                         env = 'dev',
+                         projectId
+                       }: {
+    dbAlias: string,
+    env: string,
+    projectId: string
+  }) {
+    // todo: ignore meta projects
+    if (this.connectionRefs?.[projectId]?.[env]?.[dbAlias]) {
+      try {
+        const conn = this.connectionRefs[projectId][env][dbAlias];
+        conn.destroy();
+        delete this.connectionRefs[projectId][env][dbAlias];
+      } catch (e) {
+        console.log(e);
+      }
+    }
+  }
+
   public static get({
                       dbAlias = 'db',
                       env = 'dev',

packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts

@@ -15,6 +15,7 @@ import {
 } from 'nc-help'
 import slash from 'slash';
 import {v4 as uuidv4} from 'uuid';
+import {ncp} from 'ncp';
 
 import IEmailAdapter from "../../../interface/IEmailAdapter";
 import IStorageAdapter from "../../../interface/IStorageAdapter";
@@ -37,6 +38,7 @@ import {RestApiBuilder} from "../rest/RestApiBuilder";
 import RestAuthCtrl from "../rest/RestAuthCtrlEE";
 import {packageVersion} from 'nc-help';
 import NcMetaIO, {META_TABLES} from "./NcMetaIO";
+import {promisify} from "util";
 
 const XC_PLUGIN_DET = 'XC_PLUGIN_DET';
 
@@ -303,7 +305,6 @@ const result = {
     }
 
 
-
     return res.json(result);
   }
 
@@ -352,7 +353,11 @@ const result = {
           const data = JSON.parse(fs.readFileSync(path.join(metaFolder, `${tn}.json`), 'utf8'));
           for (const row of data) {
             delete row.id;
-            await this.xcMeta.metaInsert(projectId, dbAlias, tn, row)
+            await this.xcMeta.metaInsert(projectId, dbAlias, tn, {
+              ...row,
+              db_alias: dbAlias,
+              project_id: projectId
+            })
           }
         }
       }
@@ -376,6 +381,98 @@ const result = {
   // NOTE: xc-meta
   // Extract and import metadata and config from zip file
   public async xcMetaTablesImportZipToLocalFsAndDb(args, file, req) {
+    try {
+      await this.xcMetaTablesReset(args);
+      let projectConfigPath;
+      // let storeFilePath;
+      await extract(file.path, {
+        dir: path.join(this.config.toolDir, 'uploads'),
+        onEntry(entry, _zipfile) {
+          // extract xc_project.json file path
+          if (entry.fileName?.endsWith('nc_project.json')) {
+            projectConfigPath = entry.fileName;
+          }
+        }
+      });
+      // delete temporary upload file
+      fs.unlinkSync(file.path);
+
+      let projectId = this.getProjectId(args)
+      if (!projectConfigPath) {
+        throw new Error('Missing project config file')
+      }
+
+      const projectDetailsJSON: any = fs.readFileSync(path.join(this.config.toolDir, 'uploads', projectConfigPath), 'utf8');
+      const projectDetails = projectDetailsJSON && JSON.parse(projectDetailsJSON);
+
+      if (args.args.importsToCurrentProject) {
+        await promisify(ncp)(path.join(this.config.toolDir, 'uploads', 'nc', projectDetails.id), path.join(this.config.toolDir, 'nc', projectId), {clobber:true})
+      } else {
+        // decrypt with old key and encrypt again with latest key
+        const projectConfig = JSON.parse(CryptoJS.AES.decrypt(projectDetails.config, projectDetails.key).toString(CryptoJS.enc.Utf8))
+        // delete projectDetails.key;
+        projectDetails.config = projectConfig;
+
+
+        // create new project and import
+        const project = await this.xcMeta.projectCreate(projectDetails.title, projectConfig, projectDetails.description);
+        projectId = project.id;
+
+        // move files to newly created project meta folder
+        await promisify(ncp)(path.join(this.config.toolDir, 'uploads', 'nc', projectDetails.id), path.join(this.config.toolDir, 'nc', projectId))
+
+        await this.xcMeta.projectAddUser(projectId, req?.session?.passport?.user?.id, 'owner,creator');
+        await this.projectMgr.getSqlMgr({
+          ...projectConfig,
+          metaDb: this.xcMeta?.knex
+        }).projectOpenByWeb(projectConfig);
+        this.projectConfigs[projectId] = projectConfig;
+        args.freshImport = true;
+      }
+
+
+      //   const importProjectId = projectConfig?.id;
+      //
+      //   // check project already exist
+      //   if (await this.xcMeta.projectGetById(importProjectId)) {
+      //     // todo:
+      //     throw new Error(`Project with id '${importProjectId}' already exist, it's not allowed at the moment`)
+      //   } else {
+      //     // create the project if not found
+      //     await this.xcMeta.knex('nc_projects').insert(projectConfig);
+      //     projectConfig = JSON.parse((await this.xcMeta.projectGetById(importProjectId))?.config);
+      //
+      //     // duplicated code from project create - see projectCreateByWeb
+      //     await this.xcMeta.projectAddUser(importProjectId, req?.session?.passport?.user?.id, 'owner,creator');
+      //     await this.projectMgr.getSqlMgr({
+      //       ...projectConfig,
+      //       metaDb: this.xcMeta?.knex
+      //     }).projectOpenByWeb(projectConfig);
+      //     this.projectConfigs[importProjectId] = projectConfig;
+      //
+      //     args.freshImport = true;
+      //   }
+      //   args.project_id = importProjectId;
+      // }
+
+      args.project_id = projectId
+
+      await this.xcMetaTablesImportLocalFsToDb(args, req);
+      this.xcMeta.audit(projectId, null, 'nc_audit', {
+        op_type: 'META',
+        op_sub_type: 'IMPORT_FROM_ZIP',
+        user: req.user.email,
+        description: `imported ${projectId} from zip file uploaded `, ip: req.clientIp
+      })
+
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  // NOTE: xc-meta
+  // Extract and import metadata and config from zip file
+  public async xcMetaTablesImportZipToLocalFsAndDbV1(args, file, req) {
     try {
       await this.xcMetaTablesReset(args);
       let projectConfigPath;