feat: test apis

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb-nest/src/modules/test/TestResetService/index.ts

@@ -0,0 +1,195 @@
+import axios from 'axios';
+import Project from '../../../models/Project';
+import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
+import Noco from '../../../Noco';
+import User from '../../../models/User';
+import NocoCache from '../../../cache/NocoCache';
+import { CacheScope } from '../../../utils/globals';
+import ProjectUser from '../../../models/ProjectUser';
+import resetPgSakilaProject from './resetPgSakilaProject';
+import resetMysqlSakilaProject from './resetMysqlSakilaProject';
+import resetMetaSakilaSqliteProject from './resetMetaSakilaSqliteProject';
+
+const workerStatus = {};
+
+const loginRootUser = async () => {
+  const response = await axios.post(
+    'http://localhost:8080/api/v1/auth/user/signin',
+    { email: 'user@nocodb.com', password: 'Password123.' }
+  );
+
+  return response.data.token;
+};
+
+const projectTitleByType = {
+  sqlite: 'sampleREST',
+  mysql: 'externalREST',
+  pg: 'pgExtREST',
+};
+
+export class TestResetService {
+  private readonly parallelId;
+  // todo: Hack to resolve issue with pg resetting
+  private readonly workerId;
+  private readonly dbType;
+  private readonly isEmptyProject: boolean;
+
+  constructor({
+    parallelId,
+    dbType,
+    isEmptyProject,
+    workerId,
+  }: {
+    parallelId: string;
+    dbType: string;
+    isEmptyProject: boolean;
+    workerId: string;
+  }) {
+    this.parallelId = parallelId;
+    this.dbType = dbType;
+    this.isEmptyProject = isEmptyProject;
+    this.workerId = workerId;
+  }
+
+  async process() {
+    try {
+      // console.log(
+      //   `earlier workerStatus: parrelledId: ${this.parallelId}:`,
+      //   workerStatus[this.parallelId]
+      // );
+
+      // wait till previous worker is done
+      while (workerStatus[this.parallelId] === 'processing') {
+        console.log(
+          `waiting for previous worker to finish parrelelId:${this.parallelId}`
+        );
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+      }
+
+      workerStatus[this.parallelId] = 'processing';
+
+      const token = await loginRootUser();
+
+      const { project } = await this.resetProject({
+        token,
+        dbType: this.dbType,
+        parallelId: this.parallelId,
+        workerId: this.workerId,
+      });
+
+      try {
+        await removeAllProjectCreatedByTheTest(this.parallelId);
+        await removeAllPrefixedUsersExceptSuper(this.parallelId);
+      } catch (e) {
+        console.log(`Error in cleaning up project: ${this.parallelId}`, e);
+      }
+
+      workerStatus[this.parallelId] = 'completed';
+      return { token, project };
+    } catch (e) {
+      console.error('TestResetService:process', e);
+      workerStatus[this.parallelId] = 'errored';
+      return { error: e };
+    }
+  }
+
+  async resetProject({
+    token,
+    dbType,
+    parallelId,
+    workerId,
+  }: {
+    token: string;
+    dbType: string;
+    parallelId: string;
+    workerId: string;
+  }) {
+    const title = `${projectTitleByType[dbType]}${parallelId}`;
+    const project: Project | undefined = await Project.getByTitle(title);
+
+    if (project) {
+      await removeProjectUsersFromCache(project);
+
+      const bases = await project.getBases();
+
+      for (const base of bases) {
+        await NcConnectionMgrv2.deleteAwait(base);
+        await base.delete(Noco.ncMeta, { force: true });
+      }
+
+      await Project.delete(project.id);
+    }
+
+    if (dbType == 'sqlite') {
+      await resetMetaSakilaSqliteProject({
+        token,
+        title,
+        parallelId,
+        isEmptyProject: this.isEmptyProject,
+      });
+    } else if (dbType == 'mysql') {
+      await resetMysqlSakilaProject({
+        token,
+        title,
+        parallelId,
+        oldProject: project,
+        isEmptyProject: this.isEmptyProject,
+      });
+    } else if (dbType == 'pg') {
+      await resetPgSakilaProject({
+        token,
+        title,
+        parallelId: workerId,
+        oldProject: project,
+        isEmptyProject: this.isEmptyProject,
+      });
+    }
+
+    return {
+      project: await Project.getByTitle(title),
+    };
+  }
+}
+
+const removeAllProjectCreatedByTheTest = async (parallelId: string) => {
+  const projects = await Project.list({});
+
+  for (const project of projects) {
+    if (project.title.startsWith(`nc_test_${parallelId}_`)) {
+      await Project.delete(project.id);
+    }
+  }
+};
+
+const removeAllPrefixedUsersExceptSuper = async (parallelId: string) => {
+  const users = (await User.list()).filter(
+    (user) => !user.roles.includes('super')
+  );
+
+  for (const user of users) {
+    if (user.email.startsWith(`nc_test_${parallelId}_`)) {
+      await NocoCache.del(`${CacheScope.USER}:${user.email}`);
+      await User.delete(user.id);
+    }
+  }
+};
+
+// todo: Remove this once user deletion improvement PR is merged
+const removeProjectUsersFromCache = async (project: Project) => {
+  const projectUsers: ProjectUser[] = await ProjectUser.getUsersList({
+    project_id: project.id,
+    limit: 1000,
+    offset: 0,
+  });
+
+  for (const projectUser of projectUsers) {
+    try {
+      const user: User = await User.get(projectUser.fk_user_id);
+      await NocoCache.del(
+        `${CacheScope.PROJECT_USER}:${project.id}:${user.id}`
+      );
+    } catch (e) {
+      console.error('removeProjectUsersFromCache', e);
+    }
+  }
+};

packages/nocodb-nest/src/modules/test/TestResetService/resetMetaSakilaSqliteProject.ts

@@ -0,0 +1,92 @@
+import { promises as fs } from 'fs';
+import axios from 'axios';
+
+const sqliteFilePath = (parallelId: string) => {
+  const rootDir = __dirname.replace(
+    '/src/lib/services/test/TestResetService',
+    '',
+  );
+
+  return `${rootDir}/test_sakila_${parallelId}.db`;
+};
+
+const sakilaProjectConfig = (title: string, parallelId: string) => ({
+  title,
+  bases: [
+    {
+      type: 'sqlite3',
+      config: {
+        client: 'sqlite3',
+        connection: {
+          client: 'sqlite3',
+          connection: {
+            filename: sqliteFilePath(parallelId),
+            database: 'test_sakila',
+            multipleStatements: true,
+          },
+        },
+      },
+      inflection_column: 'camelize',
+      inflection_table: 'camelize',
+    },
+  ],
+  external: true,
+});
+
+const resetMetaSakilaSqliteProject = async ({
+  parallelId,
+  token,
+  title,
+  isEmptyProject,
+}: {
+  parallelId: string;
+  token: string;
+  title: string;
+  isEmptyProject: boolean;
+}) => {
+  await deleteSqliteFileIfExists(parallelId);
+
+  if (!isEmptyProject) await seedSakilaSqliteFile(parallelId);
+
+  await createProject(token, title, parallelId);
+};
+
+const createProject = async (
+  token: string,
+  title: string,
+  parallelId: string,
+) => {
+  const response = await axios.post(
+    'http://localhost:8080/api/v1/db/meta/projects/',
+    sakilaProjectConfig(title, parallelId),
+    {
+      headers: {
+        'xc-auth': token,
+      },
+    },
+  );
+  if (response.status !== 200) {
+    console.error('Error creating project', response.data);
+  }
+  return response.data;
+};
+
+const deleteSqliteFileIfExists = async (parallelId: string) => {
+  if (await fs.stat(sqliteFilePath(parallelId)).catch(() => null)) {
+    await fs.unlink(sqliteFilePath(parallelId));
+  }
+};
+
+const seedSakilaSqliteFile = async (parallelId: string) => {
+  const testsDir = __dirname.replace(
+    '/src/lib/services/test/TestResetService',
+    '/tests',
+  );
+
+  await fs.copyFile(
+    `${testsDir}/sqlite-sakila-db/sakila.db`,
+    sqliteFilePath(parallelId),
+  );
+};
+
+export default resetMetaSakilaSqliteProject;

packages/nocodb-nest/src/modules/test/TestResetService/resetMysqlSakilaProject.ts

@@ -0,0 +1,202 @@
+import { promises as fs } from 'fs';
+import axios from 'axios';
+import { knex } from 'knex';
+import Audit from '../../../models/Audit';
+import type { Knex } from 'knex';
+import type Project from '../../../models/Project';
+
+const config = {
+  client: 'mysql2',
+  connection: {
+    host: 'localhost',
+    port: 3306,
+    user: 'root',
+    password: 'password',
+    database: 'sakila',
+    multipleStatements: true,
+    dateStrings: true,
+  },
+};
+
+const extMysqlProject = (title, parallelId) => ({
+  title,
+  bases: [
+    {
+      type: 'mysql2',
+      config: {
+        client: 'mysql2',
+        connection: {
+          host: 'localhost',
+          port: '3306',
+          user: 'root',
+          password: 'password',
+          database: `test_sakila_${parallelId}`,
+        },
+      },
+      inflection_column: 'camelize',
+      inflection_table: 'camelize',
+    },
+  ],
+  external: true,
+});
+
+const isSakilaMysqlToBeReset = async (
+  knex: Knex,
+  parallelId: string,
+  project?: Project,
+) => {
+  const tablesInDbInfo: Array<any> = await knex.raw(
+    `SELECT table_name FROM information_schema.tables WHERE table_schema = 'test_sakila_${parallelId}'`,
+  );
+
+  const nonMetaTablesInDb = tablesInDbInfo[0]
+    .map((t) => t['TABLE_NAME'])
+    .filter((table) => table !== 'nc_evolutions');
+
+  const mysqlSakilaTablesAndViews = [
+    ...mysqlSakilaTables,
+    ...mysqlSakilaSqlViews,
+  ];
+
+  if (
+    nonMetaTablesInDb.length === 0 ||
+    // If there are sakila tables
+    !nonMetaTablesInDb.includes(`actor`) ||
+    // If there are no pg sakila tables in tables in db
+    !(
+      nonMetaTablesInDb.length === mysqlSakilaTablesAndViews.length &&
+      nonMetaTablesInDb.every((t) => mysqlSakilaTablesAndViews.includes(t))
+    )
+  ) {
+    return true;
+  }
+
+  if (!project) return true;
+
+  const audits = await Audit.projectAuditList(project.id, {});
+
+  // todo: Will be fixed in the data resetting revamp
+  console.log(`audits:resetMysqlSakilaProject:${parallelId}`, audits?.length);
+  return true;
+};
+
+const resetSakilaMysql = async (
+  knex: Knex,
+  parallelId: string,
+  isEmptyProject: boolean,
+) => {
+  const testsDir = __dirname.replace(
+    '/src/lib/services/test/TestResetService',
+    '/tests',
+  );
+
+  try {
+    await knex.raw(`DROP DATABASE test_sakila_${parallelId}`);
+  } catch (e) {
+    console.log('Error dropping db', e);
+  }
+  await knex.raw(`CREATE DATABASE test_sakila_${parallelId}`);
+
+  if (isEmptyProject) return;
+
+  const trx = await knex.transaction();
+
+  try {
+    const schemaFile = await fs.readFile(
+      `${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql`,
+    );
+    const dataFile = await fs.readFile(
+      `${testsDir}/mysql-sakila-db/04-test-sakila-data.sql`,
+    );
+
+    await trx.raw(
+      schemaFile
+        .toString()
+        .replace(/test_sakila/g, `test_sakila_${parallelId}`),
+    );
+    await trx.raw(
+      dataFile.toString().replace(/test_sakila/g, `test_sakila_${parallelId}`),
+    );
+
+    await trx.commit();
+  } catch (e) {
+    console.log('Error resetting mysql db', e);
+    await trx.rollback(e);
+  }
+};
+
+const resetMysqlSakilaProject = async ({
+  token,
+  title,
+  parallelId,
+  oldProject,
+  isEmptyProject,
+}: {
+  token: string;
+  title: string;
+  parallelId: string;
+  oldProject?: Project | undefined;
+  isEmptyProject: boolean;
+}) => {
+  const nc_knex = knex(config);
+
+  try {
+    await nc_knex.raw(`USE test_sakila_${parallelId}`);
+  } catch (e) {
+    await nc_knex.raw(`CREATE DATABASE test_sakila_${parallelId}`);
+    await nc_knex.raw(`USE test_sakila_${parallelId}`);
+  }
+
+  if (
+    isEmptyProject ||
+    (await isSakilaMysqlToBeReset(nc_knex, parallelId, oldProject))
+  ) {
+    await resetSakilaMysql(nc_knex, parallelId, isEmptyProject);
+  }
+
+  const response = await axios.post(
+    'http://localhost:8080/api/v1/db/meta/projects/',
+    extMysqlProject(title, parallelId),
+    {
+      headers: {
+        'xc-auth': token,
+      },
+    },
+  );
+  if (response.status !== 200) {
+    console.error('Error creating project', response.data);
+  }
+
+  await nc_knex.destroy();
+};
+
+const mysqlSakilaTables = [
+  'actor',
+  'address',
+  'category',
+  'city',
+  'country',
+  'customer',
+  'film',
+  'film_text',
+  'film_actor',
+  'film_category',
+  'inventory',
+  'language',
+  'payment',
+  'rental',
+  'staff',
+  'store',
+];
+
+const mysqlSakilaSqlViews = [
+  'actor_info',
+  'customer_list',
+  'film_list',
+  'nicer_but_slower_film_list',
+  'sales_by_film_category',
+  'sales_by_store',
+  'staff_list',
+];
+
+export default resetMysqlSakilaProject;

packages/nocodb-nest/src/modules/test/TestResetService/resetPgSakilaProject.ts

@@ -0,0 +1,193 @@
+import { promises as fs } from 'fs';
+import axios from 'axios';
+import { knex } from 'knex';
+import Audit from '../../../models/Audit';
+import type Project from '../../../models/Project';
+
+const config = {
+  client: 'pg',
+  connection: {
+    host: 'localhost',
+    port: 5432,
+    user: 'postgres',
+    password: 'password',
+    database: 'postgres',
+    multipleStatements: true,
+  },
+  searchPath: ['public', 'information_schema'],
+  pool: { min: 0, max: 5 },
+};
+
+const extMysqlProject = (title, parallelId) => ({
+  title,
+  bases: [
+    {
+      type: 'pg',
+      config: {
+        client: 'pg',
+        connection: {
+          host: 'localhost',
+          port: '5432',
+          user: 'postgres',
+          password: 'password',
+          database: `sakila_${parallelId}`,
+        },
+        searchPath: ['public'],
+      },
+      inflection_column: 'camelize',
+      inflection_table: 'camelize',
+    },
+  ],
+  external: true,
+});
+
+const isSakilaPgToBeReset = async (parallelId: string, project?: Project) => {
+  const sakilaKnex = knex(sakilaKnexConfig(parallelId));
+
+  const tablesInDb: Array<string> = (
+    await sakilaKnex.raw(
+      `SELECT * FROM information_schema.tables WHERE table_schema = 'public'`,
+    )
+  ).rows.map((row) => row.table_name);
+
+  await sakilaKnex.destroy();
+
+  const nonMetaTablesInDb = tablesInDb.filter(
+    (table) => table !== 'nc_evolutions',
+  );
+  const pgSakilaTablesAndViews = [...pgSakilaTables, ...pgSakilaSqlViews];
+
+  if (
+    tablesInDb.length === 0 ||
+    // If there are sakila tables
+    !tablesInDb.includes(`actor`) ||
+    // If there are no pg sakila tables in tables in db
+    !(
+      nonMetaTablesInDb.length === pgSakilaTablesAndViews.length &&
+      nonMetaTablesInDb.every((t) => pgSakilaTablesAndViews.includes(t))
+    )
+  ) {
+    return true;
+  }
+
+  if (!project) return false;
+
+  const audits = await Audit.projectAuditList(project.id, {});
+
+  return audits?.length > 0;
+};
+
+const resetSakilaPg = async (parallelId: string, isEmptyProject: boolean) => {
+  const testsDir = __dirname.replace(
+    '/src/lib/services/test/TestResetService',
+    '/tests',
+  );
+
+  if (isEmptyProject) return;
+
+  try {
+    const sakilaKnex = knex(sakilaKnexConfig(parallelId));
+    const schemaFile = await fs.readFile(
+      `${testsDir}/pg-sakila-db/01-postgres-sakila-schema.sql`,
+    );
+    await sakilaKnex.raw(schemaFile.toString());
+
+    const trx = await sakilaKnex.transaction();
+    const dataFile = await fs.readFile(
+      `${testsDir}/pg-sakila-db/02-postgres-sakila-insert-data.sql`,
+    );
+    await trx.raw(dataFile.toString());
+    await trx.commit();
+
+    await sakilaKnex.destroy();
+  } catch (e) {
+    console.error(`Error resetting pg sakila db: Worker ${parallelId}`);
+    throw Error(`Error resetting pg sakila db: Worker ${parallelId}`);
+  }
+};
+
+const sakilaKnexConfig = (parallelId: string) => ({
+  ...config,
+  connection: {
+    ...config.connection,
+    database: `sakila_${parallelId}`,
+  },
+});
+
+const resetPgSakilaProject = async ({
+  token,
+  title,
+  parallelId,
+  oldProject,
+  isEmptyProject,
+}: {
+  token: string;
+  title: string;
+  parallelId: string;
+  oldProject?: Project | undefined;
+  isEmptyProject: boolean;
+}) => {
+  const pgknex = knex(config);
+
+  try {
+    await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`);
+  } catch (e) {}
+
+  if (isEmptyProject || (await isSakilaPgToBeReset(parallelId, oldProject))) {
+    await pgknex.raw(`DROP DATABASE IF EXISTS sakila_${parallelId}`);
+    await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`);
+    await pgknex.destroy();
+
+    await resetSakilaPg(parallelId, isEmptyProject);
+  }
+
+  const response = await axios.post(
+    'http://localhost:8080/api/v1/db/meta/projects/',
+    extMysqlProject(title, parallelId),
+    {
+      headers: {
+        'xc-auth': token,
+      },
+    },
+  );
+  if (response.status !== 200) {
+    console.error('Error creating project', response.data);
+    throw new Error('Error creating project', response.data);
+  }
+};
+
+const pgSakilaTables = [
+  'country',
+  'city',
+  'actor',
+  'film_actor',
+  'category',
+  'film_category',
+  'language',
+  'film',
+  'payment_p2007_01',
+  'payment_p2007_02',
+  'payment_p2007_03',
+  'payment_p2007_04',
+  'payment_p2007_05',
+  'payment_p2007_06',
+  'payment',
+  'customer',
+  'inventory',
+  'rental',
+  'address',
+  'staff',
+  'store',
+];
+
+const pgSakilaSqlViews = [
+  'actor_info',
+  'customer_list',
+  'film_list',
+  'nicer_but_slower_film_list',
+  'sales_by_film_category',
+  'sales_by_store',
+  'staff_list',
+];
+
+export default resetPgSakilaProject;

packages/nocodb-nest/src/modules/test/test.controller.ts

@@ -1,7 +1,20 @@
-import { Controller } from '@nestjs/common';
+import { Controller, Post, Req } from '@nestjs/common';
 import { TestService } from './test.service';
+import { TestResetService } from './TestResetService';
 
 @Controller('test')
 export class TestController {
   constructor(private readonly testService: TestService) {}
+
+  @Post('/api/v1/meta/test/reset')
+  async reset(@Req() req) {
+    const service = new TestResetService({
+      parallelId: req.body.parallelId,
+      dbType: req.body.dbType,
+      isEmptyProject: req.body.isEmptyProject,
+      workerId: req.body.workerId,
+    });
+
+    return await service.process();
+  }
 }