refactor/Improve db config management of test env

packages/nocodb/tests/mysql-sakila-db/03-test-sakila-schema.sql

@@ -210,11 +210,10 @@ SET @@default_storage_engine = @old_default_storage_engine;
 -- Triggers for loading film_text from film
 --
 
-DELIMITER ;;
 CREATE TRIGGER `ins_film` AFTER INSERT ON `film` FOR EACH ROW BEGIN
     INSERT INTO film_text (film_id, title, description)
         VALUES (new.film_id, new.title, new.description);
-  END;;
+  END;
 
 
 CREATE TRIGGER `upd_film` AFTER UPDATE ON `film` FOR EACH ROW BEGIN
@@ -226,14 +225,12 @@ CREATE TRIGGER `upd_film` AFTER UPDATE ON `film` FOR EACH ROW BEGIN
                 film_id=new.film_id
         WHERE film_id=old.film_id;
     END IF;
-  END;;
+  END;
 
 
 CREATE TRIGGER `del_film` AFTER DELETE ON `film` FOR EACH ROW BEGIN
     DELETE FROM film_text WHERE film_id = old.film_id;
-  END;;
+  END;
-
-DELIMITER ;
 
 --
 -- Table structure for table `inventory`
@@ -472,8 +469,6 @@ GROUP BY a.actor_id, a.first_name, a.last_name;
 -- Procedure structure for procedure `rewards_report`
 --
 
-DELIMITER //
-
 CREATE PROCEDURE rewards_report (
     IN min_monthly_purchases TINYINT UNSIGNED
     , IN min_dollar_amount_purchased DECIMAL(10,2)
@@ -535,13 +530,9 @@ proc: BEGIN
 
     /* Clean up */
     DROP TABLE tmpCustomer;
-END //
+END;
-
-DELIMITER ;
 
-DELIMITER $$
+CREATE FUNCTION IF NOT EXISTS get_customer_balance(p_customer_id INT, p_effective_date DATETIME) RETURNS DECIMAL(5,2)
-
-CREATE FUNCTION get_customer_balance(p_customer_id INT, p_effective_date DATETIME) RETURNS DECIMAL(5,2)
     DETERMINISTIC
     READS SQL DATA
 BEGIN
@@ -580,11 +571,7 @@ BEGIN
     AND payment.customer_id = p_customer_id;
 
   RETURN v_rentfees + v_overfees - v_payments;
-END $$
+END;
-
-DELIMITER ;
-
-DELIMITER $$
 
 CREATE PROCEDURE film_in_stock(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT)
 READS SQL DATA
@@ -601,11 +588,7 @@ BEGIN
      AND store_id = p_store_id
      AND inventory_in_stock(inventory_id)
      INTO p_film_count;
-END $$
+END;  
-
-DELIMITER ;
-
-DELIMITER $$
 
 CREATE PROCEDURE film_not_in_stock(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT)
 READS SQL DATA
@@ -622,13 +605,10 @@ BEGIN
      AND store_id = p_store_id
      AND NOT inventory_in_stock(inventory_id)
      INTO p_film_count;
-END $$
+END;
-
-DELIMITER ;
 
-DELIMITER $$
 
-CREATE FUNCTION inventory_held_by_customer(p_inventory_id INT) RETURNS INT
+CREATE FUNCTION IF NOT EXISTS inventory_held_by_customer(p_inventory_id INT) RETURNS INT
 READS SQL DATA
 BEGIN
   DECLARE v_customer_id INT;
@@ -640,13 +620,9 @@ BEGIN
   AND inventory_id = p_inventory_id;
 
   RETURN v_customer_id;
-END $$
+END;
 
-DELIMITER ;
+CREATE FUNCTION IF NOT EXISTS inventory_in_stock(p_inventory_id INT) RETURNS BOOLEAN
-
-DELIMITER $$
-
-CREATE FUNCTION inventory_in_stock(p_inventory_id INT) RETURNS BOOLEAN
 READS SQL DATA
 BEGIN
     DECLARE v_rentals INT;
@@ -673,9 +649,7 @@ BEGIN
     ELSE
       RETURN TRUE;
     END IF;
-END $$
+END;
-
-DELIMITER ;
 
 SET SQL_MODE=@OLD_SQL_MODE;
 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;

packages/nocodb/tests/unit/TestDbMngr.ts

@@ -0,0 +1,83 @@
+import { DbConfig } from "../../src/interface/config";
+import { NcConfigFactory } from "../../src/lib";
+import SqlMgrv2 from "../../src/lib/db/sql-mgr/v2/SqlMgrv2";
+import fs from 'fs';
+
+export default class TestDbMngr {
+  public static readonly dbName = 'test_meta';
+  public static readonly sakilaDbName = 'test_sakila';
+
+  public static dbConfig: DbConfig;
+  
+  static switchToSqlite() {
+    process.env[`DATABASE_URL`] = `sqlite:///${TestDbMngr.dbName}.sqlite`;
+    TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig(
+      NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`])
+    );
+  }
+
+  static async testConnection() {
+    try {
+      return await SqlMgrv2.testConnection(TestDbMngr.dbConfig);
+    } catch (e) {
+      console.log(e);
+      return { code: -1, message: 'Connection invalid' };
+    }
+  }
+
+  static async init({
+    user = 'root',
+    password = 'password',
+    host = 'localhost',
+    port = 3306,
+    client = 'mysql2',
+  } = {}) {
+    if(!process.env[`DATABASE_URL`]){
+      process.env[`DATABASE_URL`] = `${client}://${user}:${password}@${host}:${port}/${TestDbMngr.dbName}`;
+    }
+
+    TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig(
+      NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`])
+    );
+    this.dbConfig.meta = {
+      tn: 'nc_evolutions',
+      dbAlias: 'db',
+      api: {
+        type: 'rest',
+        prefix: '',
+        graphqlDepthLimit: 10,
+      },
+      inflection: {
+        tn: 'camelize',
+        cn: 'camelize',
+      },
+    }
+
+    const result = await TestDbMngr.testConnection()
+    if(result.code === -1){
+      TestDbMngr.switchToSqlite();
+    }
+  }
+
+  static getMetaDbConfig() {
+    return TestDbMngr.dbConfig;
+  }
+
+  static getSakilaDbConfig() {
+    const sakilaDbConfig = { ...TestDbMngr.dbConfig };
+    sakilaDbConfig.connection.database = TestDbMngr.sakilaDbName;
+    sakilaDbConfig.connection.multipleStatements = true
+
+    return sakilaDbConfig;
+  }
+
+  static async seedSakila(sakilaKnexClient) {     
+    const testsDir = __dirname.replace('tests/unit', 'tests');
+    const schemaFile = fs.readFileSync(`${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql`).toString();
+    const dataFile = fs.readFileSync(`${testsDir}/mysql-sakila-db/04-test-sakila-data.sql`).toString();
+
+    await sakilaKnexClient.raw(schemaFile);
+    await sakilaKnexClient.raw(dataFile);
+  }
+
+}

packages/nocodb/tests/unit/dbConfig.ts

@@ -1,55 +0,0 @@
-import { NcConfigFactory } from "../../src/lib";
-
-
-const sakilaTableNames = [
-  'actor',
-  'address',
-  'category',
-  'city',
-  'country',
-  'customer',
-  'film',
-  'film_actor',
-  'film_category',
-  'film_text',
-  'inventory',
-  'language',
-  'payment',
-  'rental',
-  'staff',
-  'store',
-  'actor_info',
-  'customer_list',
-  'film_list',
-  'nicer_but_slower_film_list',
-  'sales_by_film_category',
-  'sales_by_store',
-  'staff_list',
-];
-
-const dbName = 'test_meta';
-const sakilaDbName = 'test_sakila';
-const dbUser = 'root';
-const dbPassword = 'password';
-
-process.env[`DATABASE_URL`] = `mysql2://${dbUser}:${dbPassword}@localhost:3306/${dbName}`;
-
-const dbConfig = NcConfigFactory.urlToDbConfig(
-  NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`])
-);
-dbConfig.connection.database = dbName;
-dbConfig.meta = {
-  tn: 'nc_evolutions',
-  dbAlias: 'db',
-  api: {
-    type: 'rest',
-    prefix: '',
-    graphqlDepthLimit: 10,
-  },
-  inflection: {
-    tn: 'camelize',
-    column_name: 'camelize',
-  },
-} as any;
-
-export { dbConfig, dbName, sakilaTableNames, sakilaDbName, dbUser, dbPassword };

packages/nocodb/tests/unit/factory/project.ts

@@ -1,6 +1,6 @@
 import request from 'supertest';
 import Project from '../../../src/lib/models/Project';
-import { sakilaDbName } from '../dbConfig';
+import TestDbMngr from '../TestDbMngr';
 
 const externalProjectConfig = {
   title: 'sakila',
@@ -14,7 +14,7 @@ const externalProjectConfig = {
           port: '3306',
           user: 'root',
           password: 'password',
-          database: sakilaDbName,
+          database: TestDbMngr.sakilaDbName,
         },
       },
       inflection_column: 'camelize',

packages/nocodb/tests/unit/index.test.ts

@@ -1,24 +1,19 @@
 import 'mocha';
 
 import knex from 'knex';
-import { dbName } from './dbConfig';
+
 import restTests from './rest/index.test';
 import modelTests from './model/index.test';
+import TestDbMngr from './TestDbMngr'
 
 process.env.NODE_ENV = 'test';
 process.env.TEST = 'test';
 process.env.NC_DISABLE_CACHE = 'true';
+process.env.NC_DISABLE_TELE = 'true';
 
 const setupTestMetaDb = async () => {
-  const knexClient = knex({
+  const knexClient = knex(TestDbMngr.getMetaDbConfig());
-    client: 'mysql2',
+  const dbName = TestDbMngr.dbName;
-    connection: {
-      host: 'localhost',
-      port: 3306,
-      user: 'root',
-      password: 'password',
-    },
-  });
   
   try {
     await knexClient.raw(`DROP DATABASE ${dbName}`);
@@ -28,6 +23,8 @@ const setupTestMetaDb = async () => {
 }
 
 (async function() {
+  await TestDbMngr.init();
+
   await setupTestMetaDb();
 
   modelTests();

packages/nocodb/tests/unit/init/cleanupSakila.ts

@@ -1,19 +1,7 @@
 import Audit from '../../../src/lib/models/Audit';
 import Project from '../../../src/lib/models/Project';
-import { dbPassword, dbUser, sakilaTableNames, sakilaDbName } from '../dbConfig';
-import { exec } from 'child_process';
 
-async function sh(cmd) {
+import TestDbMngr from '../TestDbMngr';
-  return new Promise(function (resolve, reject) {
-    exec(cmd, (err, stdout, stderr) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve({ stdout, stderr });
-      }
-    });
-  });
-}
 
 const dropTablesOfSakila = async (sakilaKnexClient) => {
   await sakilaKnexClient.raw('SET FOREIGN_KEY_CHECKS = 0');
@@ -30,11 +18,9 @@ const resetAndSeedSakila = async (sakilaKnexClient) => {
   try {
     await dropTablesOfSakila(sakilaKnexClient);
     
-    const testsDir = __dirname.replace('tests/unit/init', 'tests');
+    await TestDbMngr.seedSakila(sakilaKnexClient);
-    await sh(`echo "SOURCE ${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql" | mysql -u ${dbUser} -p${dbPassword} ${sakilaDbName}`);
-    await sh(`echo "SOURCE ${testsDir}/mysql-sakila-db/04-test-sakila-data.sql" | mysql -u ${dbUser} -p${dbPassword} ${sakilaDbName}`);
     
-    await sakilaKnexClient.raw(`USE ${sakilaDbName}`);
+    await sakilaKnexClient.raw(`USE ${TestDbMngr.sakilaDbName}`);
   } catch (e) {
     console.error('resetSakila', e);
     throw e
@@ -72,4 +58,30 @@ const cleanUpSakila = async (sakilaKnexClient) => {
   }
 };
 
+const sakilaTableNames = [
+  'actor',
+  'address',
+  'category',
+  'city',
+  'country',
+  'customer',
+  'film',
+  'film_actor',
+  'film_category',
+  'film_text',
+  'inventory',
+  'language',
+  'payment',
+  'rental',
+  'staff',
+  'store',
+  'actor_info',
+  'customer_list',
+  'film_list',
+  'nicer_but_slower_film_list',
+  'sales_by_film_category',
+  'sales_by_store',
+  'staff_list',
+];
+
 export { cleanUpSakila, resetAndSeedSakila };

packages/nocodb/tests/unit/init/index.ts

@@ -3,13 +3,13 @@ import express from 'express';
 import knex from 'knex';
 import { Noco } from '../../../src/lib';
 
-import { dbConfig, dbName, sakilaDbName } from '../dbConfig';
 import cleanupMeta from './cleanupMeta';
 import {cleanUpSakila, resetAndSeedSakila} from './cleanupSakila';
 import { createUser } from '../factory/user';
+import TestDbMngr from '../TestDbMngr';
 
 let server;
-const knexClient = knex(dbConfig);
+let knexClient;
 let sakilaKnexClient;
 
 const serverInit = async () => {
@@ -23,9 +23,9 @@ const resetDatabase = async () => {
   try {
     if (!Noco.initialized) {
       try {
-        await knexClient.raw(`DROP DATABASE ${dbName}`);
+        await knexClient.raw(`DROP DATABASE ${TestDbMngr.dbName}`);
       } catch (e) {}
-      await knexClient.raw(`CREATE DATABASE ${dbName}`);
+      await knexClient.raw(`CREATE DATABASE ${TestDbMngr.dbName}`);
     }
   } catch (e) {
     console.error('resetDatabase', e);
@@ -44,32 +44,25 @@ const cleanupAllTables = async () => {
 
 const setupSakila = async () => {
   try {
-    await knexClient.raw(`DROP DATABASE ${sakilaDbName}`);
+    await knexClient.raw(`DROP DATABASE ${TestDbMngr.sakilaDbName}`);
   } catch(e) {
     console.log('setupSakila',e)
   }
-  await knexClient.raw(`CREATE DATABASE ${sakilaDbName}`);
+  await knexClient.raw(`CREATE DATABASE ${TestDbMngr.sakilaDbName}`);
-  await knexClient.raw(`USE ${dbName}`);
+  await knexClient.raw(`USE ${TestDbMngr.dbName}`);
-
+
-  sakilaKnexClient = knex({
+  sakilaKnexClient = knex(TestDbMngr.getSakilaDbConfig());
-    client: 'mysql2',
+  await sakilaKnexClient.raw(`USE ${TestDbMngr.sakilaDbName}`);
-    connection: {
-      host: 'localhost',
-      port: 3306,
-      user: 'root',
-      password: 'password',
-      database: sakilaDbName,
-      multipleStatements: true,
-    },
-  });
-  await sakilaKnexClient.raw(`USE ${sakilaDbName}`);
   await resetAndSeedSakila(sakilaKnexClient);
 }
 
 const isFirstTimeRun = () => !server
 
 export default async function () {
-  await knexClient.raw(`USE ${dbName}`);
+  if(!knexClient) {
+    knexClient = knex(TestDbMngr.dbConfig);
+  }
+  await knexClient.raw(`USE ${TestDbMngr.dbName}`);
   
   await resetDatabase();
   
@@ -82,5 +75,5 @@ export default async function () {
 
   const { token } = await createUser({ app: server }, { roles: 'editor' });
 
-  return { app: server, token, dbConfig };
+  return { app: server, token, dbConfig: TestDbMngr.dbConfig };
 }