github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

fix: deprecated TestResetService

pull/6599/head
mertmit 11 months ago
parent
587fdb7cef
commit
cb15ad7295
8 changed files with 0 additions and 768 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      packages/nocodb/src/app.module.ts
  2. 222
      packages/nocodb/src/controllers/test/TestResetService/index.ts
  3. 102
      packages/nocodb/src/controllers/test/TestResetService/resetMetaSakilaSqliteProject.ts
  4. 200
      packages/nocodb/src/controllers/test/TestResetService/resetMysqlSakilaProject.ts
  5. 191
      packages/nocodb/src/controllers/test/TestResetService/resetPgSakilaProject.ts
  6. 21
      packages/nocodb/src/controllers/test/test.controller.spec.ts
  7. 20
      packages/nocodb/src/controllers/test/test.controller.ts
  8. 9
      packages/nocodb/src/modules/test/test.module.ts

3
packages/nocodb/src/app.module.ts
Unescape View File

@ -10,7 +10,6 @@ import { GuiMiddleware } from '~/middlewares/gui/gui.middleware';
import { DatasModule } from '~/modules/datas/datas.module';
import { EventEmitterModule } from '~/modules/event-emitter/event-emitter.module';
import { AuthService } from '~/services/auth.service';
import { TestModule } from '~/modules/test/test.module';
import { GlobalModule } from '~/modules/global/global.module';
import { LocalStrategy } from '~/strategies/local.strategy';
import { AuthTokenStrategy } from '~/strategies/authtoken.strategy/authtoken.strategy';
@ -31,7 +30,6 @@ export const ceModuleConfig = {
GlobalModule,
UsersModule,
AuthModule,
...(process.env['PLAYWRIGHT_TEST'] === 'true' ? [TestModule] : []),
MetasModule,
DatasModule,
EventEmitterModule,
@ -41,7 +39,6 @@ export const ceModuleConfig = {
load: [() => appConfig],
isGlobal: true,
}),
TestModule,
],
providers: [
AuthService,

222
packages/nocodb/src/controllers/test/TestResetService/index.ts
Unescape View File

@ -1,222 +0,0 @@
import axios from 'axios';
import resetPgSakilaProject from './resetPgSakilaProject';
import resetMysqlSakilaProject from './resetMysqlSakilaProject';
import resetMetaSakilaSqliteProject from './resetMetaSakilaSqliteProject';
import type ApiToken from '~/models/ApiToken';
import Base from '~/models/Base';
// import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import Noco from '~/Noco';
import User from '~/models/User';
import NocoCache from '~/cache/NocoCache';
import { CacheDelDirection, CacheScope, MetaTable } from '~/utils/globals';
import BaseUser from '~/models/BaseUser';
const workerStatus = {};
const loginRootUser = async () => {
const response = await axios.post(
'http://localhost:8080/api/v1/auth/user/signin',
{ email: 'user@nocodb.com', password: 'Password123.' },
);
return response.data.token;
};
const baseTitleByType = {
sqlite: 'sampleREST',
mysql: 'externalREST',
pg: 'pgExtREST',
};
export class TestResetService {
private readonly parallelId;
// todo: Hack to resolve issue with pg resetting
private readonly workerId;
private readonly dbType;
private readonly isEmptyProject: boolean;
constructor({
parallelId,
dbType,
isEmptyProject,
workerId,
}: {
parallelId: string;
dbType: string;
isEmptyProject: boolean;
workerId: string;
}) {
this.parallelId = parallelId;
this.dbType = dbType;
this.isEmptyProject = isEmptyProject;
this.workerId = workerId;
}
async process() {
try {
// console.log(
// `earlier workerStatus: parrelledId: ${this.parallelId}:`,
// workerStatus[this.parallelId]
// );
// wait till previous worker is done
while (workerStatus[this.parallelId] === 'processing') {
console.log(
`waiting for previous worker to finish parrelelId:${this.parallelId}`,
);
await new Promise((resolve) => setTimeout(resolve, 1000));
}
workerStatus[this.parallelId] = 'processing';
const token = await loginRootUser();
const { base } = await this.resetProject({
token,
dbType: this.dbType,
parallelId: this.parallelId,
workerId: this.workerId,
});
try {
await removeAllProjectCreatedByTheTest(this.parallelId);
await removeAllPrefixedUsersExceptSuper(this.parallelId);
await removeAllTokensCreatedByTheTest(this.parallelId);
} catch (e) {
console.log(`Error in cleaning up base: ${this.parallelId}`, e);
}
workerStatus[this.parallelId] = 'completed';
return { token, base };
} catch (e) {
console.error('TestResetService:process', e);
workerStatus[this.parallelId] = 'errored';
return { error: e };
}
}
async resetProject({
token,
dbType,
parallelId,
workerId,
}: {
token: string;
dbType: string;
parallelId: string;
workerId: string;
}) {
const title = `${baseTitleByType[dbType]}${parallelId}`;
const base: Base | undefined = await Base.getByTitle(title);
if (base) {
await this.removeProjectUsersFromCache(base);
// Kludge: Soft reset to support PG as root DB in PW tests
// Revisit to fix this later
// const sources = await base.getBases();
//
// for (const base of sources) {
// await NcConnectionMgrv2.deleteAwait(base);
// await base.delete(Noco.ncMeta, { force: true });
// }
//
// await Base.delete(base.id);
await Base.softDelete(base.id);
}
if (dbType == 'sqlite') {
await resetMetaSakilaSqliteProject({
token,
title,
parallelId,
isEmptyProject: this.isEmptyProject,
});
} else if (dbType == 'mysql') {
await resetMysqlSakilaProject({
token,
title,
parallelId,
oldProject: base,
isEmptyProject: this.isEmptyProject,
});
} else if (dbType == 'pg') {
await resetPgSakilaProject({
token,
title,
parallelId: workerId,
oldProject: base,
isEmptyProject: this.isEmptyProject,
});
}
return {
base: await Base.getByTitle(title),
};
}
// todo: Remove this once user deletion improvement PR is merged
removeProjectUsersFromCache = async (base: Base) => {
const baseUsers = await BaseUser.getUsersList({
base_id: base.id,
limit: 1000,
offset: 0,
});
for (const baseUser of baseUsers) {
try {
const user: User = (await User.get(baseUser.id)) as any;
await NocoCache.del(`${CacheScope.PROJECT_USER}:${base.id}:${user.id}`);
} catch (e) {
console.error('removeProjectUsersFromCache', e);
}
}
};
}
const removeAllProjectCreatedByTheTest = async (parallelId: string) => {
const bases = await Base.list({});
for (const base of bases) {
if (base.title.startsWith(`nc_test_${parallelId}_`)) {
await Base.delete(base.id);
}
}
};
const removeAllPrefixedUsersExceptSuper = async (parallelId: string) => {
const users = (await User.list()).filter(
(user) => !user.roles.includes('super'),
);
for (const user of users) {
if (user.email.startsWith(`nc_test_${parallelId}_`)) {
await NocoCache.del(`${CacheScope.USER}:${user.email}`);
await User.delete(user.id);
}
}
};
const removeAllTokensCreatedByTheTest = async (parallelId: string) => {
const tokens: ApiToken[] = await Noco.ncMeta.metaList(
null,
null,
MetaTable.API_TOKENS,
);
for (const token of tokens) {
if (token.description.startsWith(`nc_test_${parallelId}`)) {
await NocoCache.deepDel(
CacheScope.API_TOKEN,
`${CacheScope.API_TOKEN}:${token.token}`,
CacheDelDirection.CHILD_TO_PARENT,
);
await Noco.ncMeta.metaDelete(null, null, MetaTable.API_TOKENS, {
token: token.token,
});
}
}
};

102
packages/nocodb/src/controllers/test/TestResetService/resetMetaSakilaSqliteProject.ts
Unescape View File

@ -1,102 +0,0 @@
import { promises as fs } from 'fs';
import path from 'path';
import axios from 'axios';
import type { AxiosResponse } from 'axios';
const sqliteFilePath = (parallelId: string) => {
const rootDir = process.cwd();
return `${rootDir}/test_sakila_${parallelId}.db`;
};
const sakilaProjectConfig = (title: string, parallelId: string) => ({
title,
sources: [
{
type: 'sqlite3',
config: {
client: 'sqlite3',
connection: {
client: 'sqlite3',
connection: {
filename: sqliteFilePath(parallelId),
database: 'test_sakila',
multipleStatements: true,
},
},
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
],
external: true,
});
const resetMetaSakilaSqliteProject = async ({
parallelId,
token,
title,
isEmptyProject,
}: {
parallelId: string;
token: string;
title: string;
isEmptyProject: boolean;
}) => {
await deleteSqliteFileIfExists(parallelId);
if (!isEmptyProject) await seedSakilaSqliteFile(parallelId);
await createProject(token, title, parallelId, isEmptyProject);
};
const createProject = async (
token: string,
title: string,
parallelId: string,
isEmptyProject: boolean,
) => {
let response: AxiosResponse;
if (isEmptyProject) {
response = await axios.post(
'http://localhost:8080/api/v1/meta/bases/',
{ title },
{
headers: {
'xc-auth': token,
},
},
);
} else {
response = await axios.post(
'http://localhost:8080/api/v1/meta/bases/',
sakilaProjectConfig(title, parallelId),
{
headers: {
'xc-auth': token,
},
},
);
}
if (response.status !== 200) {
console.error('Error creating base', response.data);
}
return response.data;
};
const deleteSqliteFileIfExists = async (parallelId: string) => {
if (await fs.stat(sqliteFilePath(parallelId)).catch(() => null)) {
await fs.unlink(sqliteFilePath(parallelId));
}
};
const seedSakilaSqliteFile = async (parallelId: string) => {
const testsDir = path.join(process.cwd(), 'tests');
await fs.copyFile(
`${testsDir}/sqlite-sakila-db/sakila.db`,
sqliteFilePath(parallelId),
);
};
export default resetMetaSakilaSqliteProject;

200
packages/nocodb/src/controllers/test/TestResetService/resetMysqlSakilaProject.ts
Unescape View File

@ -1,200 +0,0 @@
import { promises as fs } from 'fs';
import path from 'path';
import axios from 'axios';
import { knex } from 'knex';
import type { Knex } from 'knex';
import type Base from '~/models/Base';
import Audit from '~/models/Audit';
const config = {
client: 'mysql2',
connection: {
host: 'localhost',
port: 3306,
user: 'root',
password: 'password',
database: 'sakila',
multipleStatements: true,
dateStrings: true,
},
};
const extMysqlProject = (title, parallelId) => ({
title,
sources: [
{
type: 'mysql2',
config: {
client: 'mysql2',
connection: {
host: 'localhost',
port: '3306',
user: 'root',
password: 'password',
database: `test_sakila_${parallelId}`,
},
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
],
external: true,
});
const isSakilaMysqlToBeReset = async (
knex: Knex,
parallelId: string,
base?: Base,
) => {
const tablesInDbInfo: Array<any> = await knex.raw(
`SELECT table_name FROM information_schema.tables WHERE table_schema = 'test_sakila_${parallelId}'`,
);
const nonMetaTablesInDb = tablesInDbInfo[0]
.map((t) => t['TABLE_NAME'])
.filter((table) => table !== 'nc_evolutions');
const mysqlSakilaTablesAndViews = [
...mysqlSakilaTables,
...mysqlSakilaSqlViews,
];
if (
nonMetaTablesInDb.length === 0 ||
// If there are sakila tables
!nonMetaTablesInDb.includes(`actor`) ||
// If there are no pg sakila tables in tables in db
!(
nonMetaTablesInDb.length === mysqlSakilaTablesAndViews.length &&
nonMetaTablesInDb.every((t) => mysqlSakilaTablesAndViews.includes(t))
)
) {
return true;
}
if (!base) return true;
const audits = await Audit.baseAuditList(base.id, {});
// todo: Will be fixed in the data resetting revamp
console.log(`audits:resetMysqlSakilaProject:${parallelId}`, audits?.length);
return true;
};
const resetSakilaMysql = async (
knex: Knex,
parallelId: string,
isEmptyProject: boolean,
) => {
const testsDir = path.join(process.cwd(), '/tests');
try {
await knex.raw(`DROP DATABASE test_sakila_${parallelId}`);
} catch (e) {
console.log('Error dropping db', e);
}
await knex.raw(`CREATE DATABASE test_sakila_${parallelId}`);
if (isEmptyProject) return;
const trx = await knex.transaction();
try {
const schemaFile = await fs.readFile(
`${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql`,
);
const dataFile = await fs.readFile(
`${testsDir}/mysql-sakila-db/04-test-sakila-data.sql`,
);
await trx.raw(
schemaFile
.toString()
.replace(/test_sakila/g, `test_sakila_${parallelId}`),
);
await trx.raw(
dataFile.toString().replace(/test_sakila/g, `test_sakila_${parallelId}`),
);
await trx.commit();
} catch (e) {
console.log('Error resetting mysql db', e);
await trx.rollback(e);
}
};
const resetMysqlSakilaProject = async ({
token,
title,
parallelId,
oldProject,
isEmptyProject,
}: {
token: string;
title: string;
parallelId: string;
oldProject?: Base | undefined;
isEmptyProject: boolean;
}) => {
const nc_knex = knex(config);
try {
await nc_knex.raw(`USE test_sakila_${parallelId}`);
} catch (e) {
await nc_knex.raw(`CREATE DATABASE test_sakila_${parallelId}`);
await nc_knex.raw(`USE test_sakila_${parallelId}`);
}
if (
isEmptyProject ||
(await isSakilaMysqlToBeReset(nc_knex, parallelId, oldProject))
) {
await resetSakilaMysql(nc_knex, parallelId, isEmptyProject);
}
const response = await axios.post(
'http://localhost:8080/api/v1/meta/bases/',
extMysqlProject(title, parallelId),
{
headers: {
'xc-auth': token,
},
},
);
if (response.status !== 200) {
console.error('Error creating base', response.data);
}
await nc_knex.destroy();
};
const mysqlSakilaTables = [
'actor',
'address',
'category',
'city',
'country',
'customer',
'film',
'film_text',
'film_actor',
'film_category',
'inventory',
'language',
'payment',
'rental',
'staff',
'store',
];
const mysqlSakilaSqlViews = [
'actor_info',
'customer_list',
'film_list',
'nicer_but_slower_film_list',
'sales_by_film_category',
'sales_by_store',
'staff_list',
];
export default resetMysqlSakilaProject;

191
packages/nocodb/src/controllers/test/TestResetService/resetPgSakilaProject.ts
Unescape View File

@ -1,191 +0,0 @@
import { promises as fs } from 'fs';
import path from 'path';
import axios from 'axios';
import { knex } from 'knex';
import type Base from '~/models/Base';
import Audit from '~/models/Audit';
const config = {
client: 'pg',
connection: {
host: 'localhost',
port: 5432,
user: 'postgres',
password: 'password',
database: 'postgres',
multipleStatements: true,
},
searchPath: ['public', 'information_schema'],
pool: { min: 0, max: 5 },
};
const extMysqlProject = (title, parallelId) => ({
title,
sources: [
{
type: 'pg',
config: {
client: 'pg',
connection: {
host: 'localhost',
port: '5432',
user: 'postgres',
password: 'password',
database: `sakila_${parallelId}`,
},
searchPath: ['public'],
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
],
external: true,
});
const isSakilaPgToBeReset = async (parallelId: string, base?: Base) => {
const sakilaKnex = knex(sakilaKnexConfig(parallelId));
const tablesInDb: Array<string> = (
await sakilaKnex.raw(
`SELECT * FROM information_schema.tables WHERE table_schema = 'public'`,
)
).rows.map((row) => row.table_name);
await sakilaKnex.destroy();
const nonMetaTablesInDb = tablesInDb.filter(
(table) => table !== 'nc_evolutions',
);
const pgSakilaTablesAndViews = [...pgSakilaTables, ...pgSakilaSqlViews];
if (
tablesInDb.length === 0 ||
// If there are sakila tables
!tablesInDb.includes(`actor`) ||
// If there are no pg sakila tables in tables in db
!(
nonMetaTablesInDb.length === pgSakilaTablesAndViews.length &&
nonMetaTablesInDb.every((t) => pgSakilaTablesAndViews.includes(t))
)
) {
return true;
}
if (!base) return false;
const audits = await Audit.baseAuditList(base.id, {});
return audits?.length > 0;
};
const resetSakilaPg = async (parallelId: string, isEmptyProject: boolean) => {
const testsDir = path.join(process.cwd(), '/tests');
if (isEmptyProject) return;
try {
const sakilaKnex = knex(sakilaKnexConfig(parallelId));
const schemaFile = await fs.readFile(
`${testsDir}/pg-sakila-db/01-postgres-sakila-schema.sql`,
);
await sakilaKnex.raw(schemaFile.toString());
const trx = await sakilaKnex.transaction();
const dataFile = await fs.readFile(
`${testsDir}/pg-sakila-db/02-postgres-sakila-insert-data.sql`,
);
await trx.raw(dataFile.toString());
await trx.commit();
await sakilaKnex.destroy();
} catch (e) {
console.error(`Error resetting pg sakila db: Worker ${parallelId}`);
throw Error(`Error resetting pg sakila db: Worker ${parallelId}`);
}
};
const sakilaKnexConfig = (parallelId: string) => ({
...config,
connection: {
...config.connection,
database: `sakila_${parallelId}`,
},
});
const resetPgSakilaProject = async ({
token,
title,
parallelId,
oldProject,
isEmptyProject,
}: {
token: string;
title: string;
parallelId: string;
oldProject?: Base | undefined;
isEmptyProject: boolean;
}) => {
const pgknex = knex(config);
try {
await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`);
} catch (e) {}
if (isEmptyProject || (await isSakilaPgToBeReset(parallelId, oldProject))) {
await pgknex.raw(`DROP DATABASE IF EXISTS sakila_${parallelId}`);
await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`);
await pgknex.destroy();
await resetSakilaPg(parallelId, isEmptyProject);
}
const response = await axios.post(
'http://localhost:8080/api/v1/meta/bases/',
extMysqlProject(title, parallelId),
{
headers: {
'xc-auth': token,
},
},
);
if (response.status !== 200) {
console.error('Error creating base', response.data);
throw new Error(response.data);
}
};
const pgSakilaTables = [
'country',
'city',
'actor',
'film_actor',
'category',
'film_category',
'language',
'film',
'payment_p2007_01',
'payment_p2007_02',
'payment_p2007_03',
'payment_p2007_04',
'payment_p2007_05',
'payment_p2007_06',
'payment',
'customer',
'inventory',
'rental',
'address',
'staff',
'store',
];
const pgSakilaSqlViews = [
'actor_info',
'customer_list',
'film_list',
'nicer_but_slower_film_list',
'sales_by_film_category',
'sales_by_store',
'staff_list',
];
export default resetPgSakilaProject;

21
packages/nocodb/src/controllers/test/test.controller.spec.ts
Unescape View File

@ -1,21 +0,0 @@
import { Test } from '@nestjs/testing';
import { TestService } from '../../modules/test/test.service';
import { TestController } from './test.controller';
import type { TestingModule } from '@nestjs/testing';
describe('TestController', () => {
let controller: TestController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [TestController],
providers: [TestService],
}).compile();
controller = module.get<TestController>(TestController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

20
packages/nocodb/src/controllers/test/test.controller.ts
Unescape View File

@ -1,20 +0,0 @@
import { Controller, HttpCode, Post, Req } from '@nestjs/common';
import { TestResetService } from '~/controllers/test/TestResetService';
@Controller()
export class TestController {
constructor() {}
@Post('/api/v1/meta/test/reset')
@HttpCode(200)
async reset(@Req() req) {
const service = new TestResetService({
parallelId: req.body.parallelId,
dbType: req.body.dbType,
isEmptyProject: req.body.isEmptyProject,
workerId: req.body.workerId,
});
return await service.process();
}
}

9
packages/nocodb/src/modules/test/test.module.ts
Unescape View File

@ -1,9 +0,0 @@
import { Module } from '@nestjs/common';
import { TestController } from '~/controllers/test/test.controller';
@Module({
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [TestController] : []),
],
})
export class TestModule {}
Write Preview
Loading…
Cancel
Save