Browse Source

feat(testing): Added postgres support

pull/3848/head
Muhammed Mustafa 2 years ago
parent
commit
0b74ba6862
  1. 65
      packages/nocodb/src/lib/services/test/TestResetService/createProjects.ts
  2. 9
      packages/nocodb/src/lib/services/test/TestResetService/index.ts
  3. 30
      packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts
  4. 54
      packages/nocodb/src/lib/services/test/TestResetService/resetMysqlSakilaProject.ts
  5. 56
      packages/nocodb/src/lib/services/test/TestResetService/resetPgSakila.ts
  6. 143
      packages/nocodb/src/lib/services/test/TestResetService/resetPgSakilaProject.ts
  7. 1710
      packages/nocodb/tests/pg-sakila-db/03-postgres-sakila-schema.sql
  8. 46702
      packages/nocodb/tests/pg-sakila-db/04-postgres-sakila-insert-data.sql
  9. 1
      scripts/playwright/.gitignore

65
packages/nocodb/src/lib/services/test/TestResetService/createProjects.ts

@ -1,65 +0,0 @@
import axios from 'axios';
// const extPgProject = {
// title: 'pgExtREST',
// bases: [
// {
// type: 'pg',
// config: {
// client: 'pg',
// connection: {
// host: 'localhost',
// port: '5432',
// user: 'postgres',
// password: 'password',
// database: 'postgres',
// },
// searchPath: ['public'],
// },
// inflection_column: 'camelize',
// inflection_table: 'camelize',
// },
// ],
// external: true,
// };
const extMysqlProject = {
title: 'externalREST',
bases: [
{
type: 'mysql2',
config: {
client: 'mysql2',
connection: {
host: 'localhost',
port: '3306',
user: 'root',
password: 'password',
database: 'test_sakila',
},
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
],
external: true,
};
const createProjects = async (token) => {
return await Promise.all(
[extMysqlProject].map(async (projectAttr) => {
const response = await axios.post(
'http://localhost:8080/api/v1/db/meta/projects/',
projectAttr,
{
headers: {
'xc-auth': token,
},
}
);
return response.data;
})
);
};
export default createProjects;

9
packages/nocodb/src/lib/services/test/TestResetService/index.ts

@ -7,6 +7,7 @@ import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
import resetMetaSakilaSqliteProject from './resetMetaSakilaSqliteProject';
import resetMysqlSakilaProject from './resetMysqlSakilaProject';
import Model from '../../../models/Model';
import resetPgSakilaProject from './resetPgSakilaProject';
const loginRootUser = async () => {
const response = await axios.post(
@ -20,6 +21,7 @@ const loginRootUser = async () => {
const projectTitleByType = {
sqlite: 'sampleREST',
mysql: 'externalREST',
pg: 'pgExtREST',
};
export class TestResetService {
@ -82,6 +84,13 @@ export class TestResetService {
parallelId,
oldProject: project,
});
} else if (dbType == 'pg') {
await resetPgSakilaProject({
token,
title,
parallelId,
oldProject: project,
});
}
return {

30
packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts

@ -1,7 +1,7 @@
import axios from 'axios';
import Knex from 'knex';
import fs from 'fs';
import { promises as fs } from 'fs';
import Audit from '../../../models/Audit';
import { sakilaTableNames } from '../../../utils/globals';
@ -61,21 +61,17 @@ const resetMetaSakilaSqlite = async (metaKnex: Knex, prefix: string) => {
const trx = await metaKnex.transaction();
try {
const schemaFile = fs
.readFileSync(
`${testsDir}/sqlite-sakila-db/03-sqlite-prefix-sakila-schema.sql`
)
.toString()
.replace(/prefix___/g, prefix);
const dataFile = fs
.readFileSync(
`${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql`
)
.toString()
.replace(/prefix___/g, prefix);
const schemaSqlQueries = schemaFile.split(';');
const schemaFile = await fs.readFile(
`${testsDir}/sqlite-sakila-db/03-sqlite-prefix-sakila-schema.sql`
);
const schemaFileStr = schemaFile.toString().replace(/prefix___/g, prefix);
const dataFile = await fs.readFile(
`${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql`
);
const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix);
const schemaSqlQueries = schemaFileStr.split(';');
for (const sqlQuery of schemaSqlQueries) {
if (sqlQuery.trim().length > 0) {
await trx.raw(
@ -86,7 +82,7 @@ const resetMetaSakilaSqlite = async (metaKnex: Knex, prefix: string) => {
}
}
const dataSqlQueries = dataFile.split(';');
const dataSqlQueries = dataFileStr.split(';');
for (const sqlQuery of dataSqlQueries) {
if (sqlQuery.trim().length > 0) {
await trx.raw(sqlQuery.trim());

54
packages/nocodb/src/lib/services/test/TestResetService/resetMysqlSakilaProject.ts

@ -1,9 +1,8 @@
import axios from 'axios';
import Knex from 'knex';
import fs from 'fs';
import { promises as fs } from 'fs';
import Audit from '../../../models/Audit';
import { sakilaTableNames } from '../../../utils/globals';
import Project from '../../../models/Project';
const config = {
@ -41,30 +40,6 @@ const extMysqlProject = (title, parallelId) => ({
external: true,
});
const mysqlSakilaSqlViews = [
'actor_info',
'customer_list',
'film_list',
'nicer_but_slower_film_list',
'sales_by_film_category',
'sales_by_store',
'staff_list',
];
const dropTablesAndViews = async (knex: Knex) => {
for (const view of mysqlSakilaSqlViews) {
try {
await knex.raw(`DROP VIEW ${view}`);
} catch (e) {}
}
for (const table of sakilaTableNames) {
try {
await knex.raw(`DROP TABLE ${table}`);
} catch (e) {}
}
};
const isSakilaMysqlToBeReset = async (
knex: Knex,
parallelId: string,
@ -89,8 +64,6 @@ const isSakilaMysqlToBeReset = async (
};
const resetSakilaMysql = async (knex: Knex, parallelId: string) => {
await dropTablesAndViews(knex);
const testsDir = __dirname.replace(
'/src/lib/services/test/TestResetService',
'/tests'
@ -106,18 +79,19 @@ const resetSakilaMysql = async (knex: Knex, parallelId: string) => {
const trx = await knex.transaction();
try {
const schemaFile = fs
.readFileSync(`${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql`)
.toString()
.replace(/test_sakila/g, `test_sakila_${parallelId}`);
const dataFile = fs
.readFileSync(`${testsDir}/mysql-sakila-db/04-test-sakila-data.sql`)
.toString()
.replace(/test_sakila/g, `test_sakila_${parallelId}`);
await trx.raw(schemaFile);
await trx.raw(dataFile);
const schemaFile = await fs.readFile(
`${testsDir}/mysql-sakila-db/03-test-sakila-schema.sql`
);
const dataFile = await fs.readFile(
`${testsDir}/mysql-sakila-db/04-test-sakila-data.sql`
);
await trx.raw(
schemaFile.toString().replace(/test_sakila/g, `test_sakila_${parallelId}`)
);
await trx.raw(
dataFile.toString().replace(/test_sakila/g, `test_sakila_${parallelId}`)
);
await trx.commit();
} catch (e) {

56
packages/nocodb/src/lib/services/test/TestResetService/resetPgSakila.ts

@ -1,56 +0,0 @@
import knex from 'knex';
import fs from 'fs';
import Project from '../../../models/Project';
import Audit from '../../../models/Audit';
const config = {
client: 'pg',
connection: {
host: 'localhost',
port: 5432,
user: 'postgres',
password: 'password',
database: 'postgres',
},
searchPath: ['public'],
meta: { dbtype: '' },
pool: { min: 0, max: 5 },
};
const isPgSakilaToBeReset = async () => {
const sakilaProject = await Project.getByTitle('pgExtREST');
const audits =
sakilaProject && (await Audit.projectAuditList(sakilaProject.id, {}));
return audits?.length > 0;
};
const resetPgSakila = async () => {
const knexClient = knex(config);
try {
await knexClient.raw(`DROP SCHEMA public CASCADE`);
} catch (e) {
console.log('Error dropping pg schema', e);
}
await knexClient.raw(`CREATE SCHEMA public`);
const testsDir = __dirname.replace(
'/src/lib/services/test/TestResetService',
'/tests'
);
const schemaFile = fs
.readFileSync(`${testsDir}/pg-sakila-db/01-postgres-sakila-schema.sql`)
.toString();
const dataFile = fs
.readFileSync(`${testsDir}/pg-sakila-db/02-postgres-sakila-insert-data.sql`)
.toString();
await knexClient.raw(schemaFile);
await knexClient.raw(dataFile);
await knexClient.destroy();
};
export { resetPgSakila, isPgSakilaToBeReset };

143
packages/nocodb/src/lib/services/test/TestResetService/resetPgSakilaProject.ts

@ -0,0 +1,143 @@
import axios from 'axios';
import Knex from 'knex';
import { promises as fs } from 'fs';
const util = require('util');
const exec = util.promisify(require('child_process').exec);
import Audit from '../../../models/Audit';
import Project from '../../../models/Project';
const config = {
client: 'pg',
connection: {
host: 'localhost',
port: 5432,
user: 'postgres',
password: 'password',
database: 'postgres',
multipleStatements: true,
},
searchPath: ['public', 'information_schema'],
pool: { min: 0, max: 5 },
};
const extMysqlProject = (title, parallelId) => ({
title,
bases: [
{
type: 'pg',
config: {
client: 'pg',
connection: {
host: 'localhost',
port: '5432',
user: 'postgres',
password: 'password',
database: `sakila_${parallelId}`,
},
searchPath: ['public'],
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
],
external: true,
});
const isSakilaPgToBeReset = async (knex: Knex, project?: Project) => {
const tablesInDb: Array<string> = (
await knex.raw(
`SELECT * FROM information_schema.tables WHERE table_schema = 'public'`
)
).rows.map((row) => row.table_name);
if (
tablesInDb.length === 0 ||
(tablesInDb.length > 0 && !tablesInDb.includes(`actor`))
) {
return true;
}
if (!project) return false;
const audits = await Audit.projectAuditList(project.id, {});
return audits?.length > 0;
};
const resetSakilaPg = async (pgknex: Knex, parallelId: string) => {
const testsDir = __dirname.replace(
'/src/lib/services/test/TestResetService',
'/tests'
);
await pgknex.raw(`DROP DATABASE IF EXISTS sakila_${parallelId}`);
await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`);
const sakilaKnex = Knex(sakilaKnexConfig(parallelId));
const schemaFile = await fs.readFile(
`${testsDir}/pg-sakila-db/03-postgres-sakila-schema.sql`
);
await sakilaKnex.raw(schemaFile.toString());
const dataFilePath = `${testsDir}/pg-sakila-db/04-postgres-sakila-insert-data.sql`;
await exec(
`export PGPASSWORD='${config.connection.password}';psql sakila_${parallelId} -h localhost -U postgres -w -f ${dataFilePath}`
);
await sakilaKnex.destroy();
};
const sakilaKnexConfig = (parallelId: string) => ({
...config,
connection: {
...config.connection,
database: `sakila_${parallelId}`,
},
});
const resetPgSakilaProject = async ({
token,
title,
parallelId,
oldProject,
}: {
token: string;
title: string;
parallelId: string;
oldProject?: Project | undefined;
}) => {
const pgknex = Knex(config);
try {
await pgknex.raw(`CREATE DATABASE sakila_${parallelId}`);
} catch (e) {}
const sakilaKnex = Knex(sakilaKnexConfig(parallelId));
if (await isSakilaPgToBeReset(sakilaKnex, oldProject)) {
await sakilaKnex.destroy();
await resetSakilaPg(pgknex, parallelId);
} else {
await sakilaKnex.destroy();
}
const response = await axios.post(
'http://localhost:8080/api/v1/db/meta/projects/',
extMysqlProject(title, parallelId),
{
headers: {
'xc-auth': token,
},
}
);
if (response.status !== 200) {
console.error('Error creating project', response.data);
}
await pgknex.destroy();
};
export default resetPgSakilaProject;

1710
packages/nocodb/tests/pg-sakila-db/03-postgres-sakila-schema.sql

File diff suppressed because it is too large Load Diff

46702
packages/nocodb/tests/pg-sakila-db/04-postgres-sakila-insert-data.sql

File diff suppressed because it is too large Load Diff

1
scripts/playwright/.gitignore vendored

@ -3,3 +3,4 @@ node_modules/
/playwright-report/
/playwright/.cache/
.env
output

Loading…
Cancel
Save