diff --git a/packages/nocodb-nest/src/modules/test/TestResetService/index.ts b/packages/nocodb-nest/src/modules/test/TestResetService/index.ts new file mode 100644 index 0000000000..045d09fb1a --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/TestResetService/index.ts @@ -0,0 +1,195 @@ +import axios from 'axios'; +import Project from '../../../models/Project'; +import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; +import Noco from '../../../Noco'; +import User from '../../../models/User'; +import NocoCache from '../../../cache/NocoCache'; +import { CacheScope } from '../../../utils/globals'; +import ProjectUser from '../../../models/ProjectUser'; +import resetPgSakilaProject from './resetPgSakilaProject'; +import resetMysqlSakilaProject from './resetMysqlSakilaProject'; +import resetMetaSakilaSqliteProject from './resetMetaSakilaSqliteProject'; + +const workerStatus = {}; + +const loginRootUser = async () => { + const response = await axios.post( + 'http://localhost:8080/api/v1/auth/user/signin', + { email: 'user@nocodb.com', password: 'Password123.' } + ); + + return response.data.token; +}; + +const projectTitleByType = { + sqlite: 'sampleREST', + mysql: 'externalREST', + pg: 'pgExtREST', +}; + +export class TestResetService { + private readonly parallelId; + // todo: Hack to resolve issue with pg resetting + private readonly workerId; + private readonly dbType; + private readonly isEmptyProject: boolean; + + constructor({ + parallelId, + dbType, + isEmptyProject, + workerId, + }: { + parallelId: string; + dbType: string; + isEmptyProject: boolean; + workerId: string; + }) { + this.parallelId = parallelId; + this.dbType = dbType; + this.isEmptyProject = isEmptyProject; + this.workerId = workerId; + } + + async process() { + try { + // console.log( + // `earlier workerStatus: parrelledId: ${this.parallelId}:`, + // workerStatus[this.parallelId] + // ); + + // wait till previous worker is done + while (workerStatus[this.parallelId] === 'processing') { + console.log( + `waiting for previous worker to finish parrelelId:${this.parallelId}` + ); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + workerStatus[this.parallelId] = 'processing'; + + const token = await loginRootUser(); + + const { project } = await this.resetProject({ + token, + dbType: this.dbType, + parallelId: this.parallelId, + workerId: this.workerId, + }); + + try { + await removeAllProjectCreatedByTheTest(this.parallelId); + await removeAllPrefixedUsersExceptSuper(this.parallelId); + } catch (e) { + console.log(`Error in cleaning up project: ${this.parallelId}`, e); + } + + workerStatus[this.parallelId] = 'completed'; + return { token, project }; + } catch (e) { + console.error('TestResetService:process', e); + workerStatus[this.parallelId] = 'errored'; + return { error: e }; + } + } + + async resetProject({ + token, + dbType, + parallelId, + workerId, + }: { + token: string; + dbType: string; + parallelId: string; + workerId: string; + }) { + const title = `${projectTitleByType[dbType]}${parallelId}`; + const project: Project | undefined = await Project.getByTitle(title); + + if (project) { + await removeProjectUsersFromCache(project); + + const bases = await project.getBases(); + + for (const base of bases) { + await NcConnectionMgrv2.deleteAwait(base); + await base.delete(Noco.ncMeta, { force: true }); + } + + await Project.delete(project.id); + } + + if (dbType == 'sqlite') { + await resetMetaSakilaSqliteProject({ + token, + title, + parallelId, + isEmptyProject: this.isEmptyProject, + }); + } else if (dbType == 'mysql') { + await resetMysqlSakilaProject({ + token, + title, + parallelId, + oldProject: project, + isEmptyProject: this.isEmptyProject, + }); + } else if (dbType == 'pg') { + await resetPgSakilaProject({ + token, + title, + parallelId: workerId, + oldProject: project, + isEmptyProject: this.isEmptyProject, + }); + } + + return { + project: await Project.getByTitle(title), + }; + } +} + +const removeAllProjectCreatedByTheTest = async (parallelId: string) => { + const projects = await Project.list({}); + + for (const project of projects) { + if (project.title.startsWith(`nc_test_${parallelId}_`)) { + await Project.delete(project.id); + } + } +}; + +const removeAllPrefixedUsersExceptSuper = async (parallelId: string) => { + const users = (await User.list()).filter( + (user) => !user.roles.includes('super') + ); + + for (const user of users) { + if (user.email.startsWith(`nc_test_${parallelId}_`)) { + await NocoCache.del(`${CacheScope.USER}:${user.email}`); + await User.delete(user.id); + } + } +}; + +// todo: Remove this once user deletion improvement PR is merged +const removeProjectUsersFromCache = async (project: Project) => { + const projectUsers: ProjectUser[] = await ProjectUser.getUsersList({ + project_id: project.id, + limit: 1000, + offset: 0, + }); + + for (const projectUser of projectUsers) { + try { + const user: User = await User.get(projectUser.fk_user_id); + await NocoCache.del( + `${CacheScope.PROJECT_USER}:${project.id}:${user.id}` + ); + } catch (e) { + console.error('removeProjectUsersFromCache', e); + } + } +}; diff --git a/packages/nocodb-nest/src/modules/test/TestResetService/resetMetaSakilaSqliteProject.ts b/packages/nocodb-nest/src/modules/test/TestResetService/resetMetaSakilaSqliteProject.ts new file mode 100644 index 0000000000..0f0989559e --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/TestResetService/resetMetaSakilaSqliteProject.ts @@ -0,0 +1,92 @@ +import { promises as fs } from 'fs'; +import axios from 'axios'; + +const sqliteFilePath = (parallelId: string) => { + const rootDir = __dirname.replace( + '/src/lib/services/test/TestResetService', + '', + ); + + return `${rootDir}/test_sakila_${parallelId}.db`; +}; + +const sakilaProjectConfig = (title: string, parallelId: string) => ({ + title, + bases: [ + { + type: 'sqlite3', + config: { + client: 'sqlite3', + connection: { + client: 'sqlite3', + connection: { + filename: sqliteFilePath(parallelId), + database: 'test_sakila', + multipleStatements: true, + }, + }, + }, + inflection_column: 'camelize', + inflection_table: 'camelize', + }, + ], + external: true, +}); + +const resetMetaSakilaSqliteProject = async ({ + parallelId, + token, + title, + isEmptyProject, +}: { + parallelId: string; + token: string; + title: string; + isEmptyProject: boolean; +}) => { + await deleteSqliteFileIfExists(parallelId); + + if (!isEmptyProject) await seedSakilaSqliteFile(parallelId); + + await createProject(token, title, parallelId); +}; + +const createProject = async ( + token: string, + title: string, + parallelId: string, +) => { + const response = await axios.post( + 'http://localhost:8080/api/v1/db/meta/projects/', + sakilaProjectConfig(title, parallelId), + { + headers: { + 'xc-auth': token, + }, + }, + ); + if (response.status !== 200) { + console.error('Error creating project', response.data); + } + return response.data; +}; + +const deleteSqliteFileIfExists = async (parallelId: string) => { + if (await fs.stat(sqliteFilePath(parallelId)).catch(() => null)) { + await fs.unlink(sqliteFilePath(parallelId)); + } +}; + +const seedSakilaSqliteFile = async (parallelId: string) => { + const testsDir = __dirname.replace( + '/src/lib/services/test/TestResetService', + '/tests', + ); + + await fs.copyFile( + `${testsDir}/sqlite-sakila-db/sakila.db`, + sqliteFilePath(parallelId), + ); +}; + +export default resetMetaSakilaSqliteProject; diff --git a/packages/nocodb-nest/src/modules/test/TestResetService/resetMysqlSakilaProject.ts b/packages/nocodb-nest/src/modules/test/TestResetService/resetMysqlSakilaProject.ts new file mode 100644 index 0000000000..7e7695db3f --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/TestResetService/resetMysqlSakilaProject.ts @@ -0,0 +1,202 @@ +import { promises as fs } from 'fs'; +import axios from 'axios'; +import { knex } from 'knex'; +import Audit from '../../../models/Audit'; +import type { Knex } from 'knex'; +import type Project from '../../../models/Project'; + +const config = { + client: 'mysql2', + connection: { + host: 'localhost', + port: 3306, + user: 'root', + password: 'password', + database: 'sakila', + multipleStatements: true, + dateStrings: true, + }, +}; + +const extMysqlProject = (title, parallelId) => ({ + title, + bases: [ + { + type: 'mysql2', + config: { + client: 'mysql2', + connection: { + host: 'localhost', + port: '3306', + user: 'root', + password: 'password', + database: `test_sakila_${parallelId}`, + }, + }, + inflection_column: 'camelize', + inflection_table: 'camelize', + }, + ], + external: true, +}); + +const isSakilaMysqlToBeReset = async ( + knex: Knex, + parallelId: string, + project?: Project, +) => { + const tablesInDbInfo: Array = await knex.raw( + `SELECT table_name FROM information_schema.tables WHERE table_schema = 'test_sakila_${parallelId}'`, + ); + + const nonMetaTablesInDb = tablesInDbInfo[0] + .map((t) => t['TABLE_NAME']) + .filter((table) => table !== 'nc_evolutions'); + + const mysqlSakilaTablesAndViews = [ + ...mysqlSakilaTables, + ...mysqlSakilaSqlViews, + ]; + + if ( + nonMetaTablesInDb.length === 0 || + // If there are sakila tables + !nonMetaTablesInDb.includes(`actor`) || + // If there are no pg sakila tables in tables in db + !( + nonMetaTablesInDb.length === mysqlSakilaTablesAndViews.length && + nonMetaTablesInDb.every((t) => mysqlSakilaTablesAndViews.includes(t)) + ) + ) { + return true; + } + + if (!project) return true; + + const audits = await Audit.projectAuditList(project.id, {}); + + // todo: Will be fixed in the data resetting revamp + console.log(`audits:resetMysqlSakilaProject:${parallelId}`, audits?.length); + return true; +}; + +const resetSakilaMysql = async ( + knex: Knex, + parallelId: string, + isEmptyProject: boolean, +) => { + const testsDir = __dirname.replace( + '/src/lib/services/test/TestResetService', + '/tests', + ); + + try { + await knex.raw(`DROP DATABASE test_sakila_${parallelId}`); + } catch (e) { + console.log('Error dropping db', e); + } + await knex.raw(`CREATE DATABASE test_sakila_${parallelId}`); + + if (isEmptyProject) return; + + const trx = await knex.transaction(); + + try { + const schemaFile = await fs.readFile( + `${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql`, + ); + const dataFile = await fs.readFile( + `${testsDir}/mysql-sakila-db/04-test-sakila-data.sql`, + ); + + await trx.raw( + schemaFile + .toString() + .replace(/test_sakila/g, `test_sakila_${parallelId}`), + ); + await trx.raw( + dataFile.toString().replace(/test_sakila/g, `test_sakila_${parallelId}`), + ); + + await trx.commit(); + } catch (e) { + console.log('Error resetting mysql db', e); + await trx.rollback(e); + } +}; + +const resetMysqlSakilaProject = async ({ + token, + title, + parallelId, + oldProject, + isEmptyProject, +}: { + token: string; + title: string; + parallelId: string; + oldProject?: Project | undefined; + isEmptyProject: boolean; +}) => { + const nc_knex = knex(config); + + try { + await nc_knex.raw(`USE test_sakila_${parallelId}`); + } catch (e) { + await nc_knex.raw(`CREATE DATABASE test_sakila_${parallelId}`); + await nc_knex.raw(`USE test_sakila_${parallelId}`); + } + + if ( + isEmptyProject || + (await isSakilaMysqlToBeReset(nc_knex, parallelId, oldProject)) + ) { + await resetSakilaMysql(nc_knex, parallelId, isEmptyProject); + } + + const response = await axios.post( + 'http://localhost:8080/api/v1/db/meta/projects/', + extMysqlProject(title, parallelId), + { + headers: { + 'xc-auth': token, + }, + }, + ); + if (response.status !== 200) { + console.error('Error creating project', response.data); + } + + await nc_knex.destroy(); +}; + +const mysqlSakilaTables = [ + 'actor', + 'address', + 'category', + 'city', + 'country', + 'customer', + 'film', + 'film_text', + 'film_actor', + 'film_category', + 'inventory', + 'language', + 'payment', + 'rental', + 'staff', + 'store', +]; + +const mysqlSakilaSqlViews = [ + 'actor_info', + 'customer_list', + 'film_list', + 'nicer_but_slower_film_list', + 'sales_by_film_category', + 'sales_by_store', + 'staff_list', +]; + +export default resetMysqlSakilaProject; diff --git a/packages/nocodb-nest/src/modules/test/TestResetService/resetPgSakilaProject.ts b/packages/nocodb-nest/src/modules/test/TestResetService/resetPgSakilaProject.ts new file mode 100644 index 0000000000..e1b94513cb --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/TestResetService/resetPgSakilaProject.ts @@ -0,0 +1,193 @@ +import { promises as fs } from 'fs'; +import axios from 'axios'; +import { knex } from 'knex'; +import Audit from '../../../models/Audit'; +import type Project from '../../../models/Project'; + +const config = { + client: 'pg', + connection: { + host: 'localhost', + port: 5432, + user: 'postgres', + password: 'password', + database: 'postgres', + multipleStatements: true, + }, + searchPath: ['public', 'information_schema'], + pool: { min: 0, max: 5 }, +}; + +const extMysqlProject = (title, parallelId) => ({ + title, + bases: [ + { + type: 'pg', + config: { + client: 'pg', + connection: { + host: 'localhost', + port: '5432', + user: 'postgres', + password: 'password', + database: `sakila_${parallelId}`, + }, + searchPath: ['public'], + }, + inflection_column: 'camelize', + inflection_table: 'camelize', + }, + ], + external: true, +}); + +const isSakilaPgToBeReset = async (parallelId: string, project?: Project) => { + const sakilaKnex = knex(sakilaKnexConfig(parallelId)); + + const tablesInDb: Array = ( + await sakilaKnex.raw( + `SELECT * FROM information_schema.tables WHERE table_schema = 'public'`, + ) + ).rows.map((row) => row.table_name); + + await sakilaKnex.destroy(); + + const nonMetaTablesInDb = tablesInDb.filter( + (table) => table !== 'nc_evolutions', + ); + const pgSakilaTablesAndViews = [...pgSakilaTables, ...pgSakilaSqlViews]; + + if ( + tablesInDb.length === 0 || + // If there are sakila tables + !tablesInDb.includes(`actor`) || + // If there are no pg sakila tables in tables in db + !( + nonMetaTablesInDb.length === pgSakilaTablesAndViews.length && + nonMetaTablesInDb.every((t) => pgSakilaTablesAndViews.includes(t)) + ) + ) { + return true; + } + + if (!project) return false; + + const audits = await Audit.projectAuditList(project.id, {}); + + return audits?.length > 0; +}; + +const resetSakilaPg = async (parallelId: string, isEmptyProject: boolean) => { + const testsDir = __dirname.replace( + '/src/lib/services/test/TestResetService', + '/tests', + ); + + if (isEmptyProject) return; + + try { + const sakilaKnex = knex(sakilaKnexConfig(parallelId)); + const schemaFile = await fs.readFile( + `${testsDir}/pg-sakila-db/01-postgres-sakila-schema.sql`, + ); + await sakilaKnex.raw(schemaFile.toString()); + + const trx = await sakilaKnex.transaction(); + const dataFile = await fs.readFile( + `${testsDir}/pg-sakila-db/02-postgres-sakila-insert-data.sql`, + ); + await trx.raw(dataFile.toString()); + await trx.commit(); + + await sakilaKnex.destroy(); + } catch (e) { + console.error(`Error resetting pg sakila db: Worker ${parallelId}`); + throw Error(`Error resetting pg sakila db: Worker ${parallelId}`); + } +}; + +const sakilaKnexConfig = (parallelId: string) => ({ + ...config, + connection: { + ...config.connection, + database: `sakila_${parallelId}`, + }, +}); + +const resetPgSakilaProject = async ({ + token, + title, + parallelId, + oldProject, + isEmptyProject, +}: { + token: string; + title: string; + parallelId: string; + oldProject?: Project | undefined; + isEmptyProject: boolean; +}) => { + const pgknex = knex(config); + + try { + await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`); + } catch (e) {} + + if (isEmptyProject || (await isSakilaPgToBeReset(parallelId, oldProject))) { + await pgknex.raw(`DROP DATABASE IF EXISTS sakila_${parallelId}`); + await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`); + await pgknex.destroy(); + + await resetSakilaPg(parallelId, isEmptyProject); + } + + const response = await axios.post( + 'http://localhost:8080/api/v1/db/meta/projects/', + extMysqlProject(title, parallelId), + { + headers: { + 'xc-auth': token, + }, + }, + ); + if (response.status !== 200) { + console.error('Error creating project', response.data); + throw new Error('Error creating project', response.data); + } +}; + +const pgSakilaTables = [ + 'country', + 'city', + 'actor', + 'film_actor', + 'category', + 'film_category', + 'language', + 'film', + 'payment_p2007_01', + 'payment_p2007_02', + 'payment_p2007_03', + 'payment_p2007_04', + 'payment_p2007_05', + 'payment_p2007_06', + 'payment', + 'customer', + 'inventory', + 'rental', + 'address', + 'staff', + 'store', +]; + +const pgSakilaSqlViews = [ + 'actor_info', + 'customer_list', + 'film_list', + 'nicer_but_slower_film_list', + 'sales_by_film_category', + 'sales_by_store', + 'staff_list', +]; + +export default resetPgSakilaProject; diff --git a/packages/nocodb-nest/src/modules/test/test.controller.ts b/packages/nocodb-nest/src/modules/test/test.controller.ts index 99725a622d..a1abd8a988 100644 --- a/packages/nocodb-nest/src/modules/test/test.controller.ts +++ b/packages/nocodb-nest/src/modules/test/test.controller.ts @@ -1,7 +1,20 @@ -import { Controller } from '@nestjs/common'; +import { Controller, Post, Req } from '@nestjs/common'; import { TestService } from './test.service'; +import { TestResetService } from './TestResetService'; @Controller('test') export class TestController { constructor(private readonly testService: TestService) {} + + @Post('/api/v1/meta/test/reset') + async reset(@Req() req) { + const service = new TestResetService({ + parallelId: req.body.parallelId, + dbType: req.body.dbType, + isEmptyProject: req.body.isEmptyProject, + workerId: req.body.workerId, + }); + + return await service.process(); + } }