github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat(testing): Improved sqlite support and added retry logic in tableUpdate in sqlite client due it popping up on high number of writes in sqlite and configured WAL mode for meta db in testing

pull/3848/head
Muhammed Mustafa 2 years ago
parent
7d8386c519
commit
01d6bd0e77
5 changed files with 135 additions and 114 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      packages/nocodb/.gitignore
  2. 206
      packages/nocodb/src/lib/db/sql-client/lib/sqlite/SqliteClient.ts
  3. 33
      packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts
  4. 4
      packages/nocodb/src/lib/utils/NcConfigFactory.ts
  5. 4
      scripts/playwright/playwright.config.ts

2
packages/nocodb/.gitignore vendored
Unescape View File

@ -18,4 +18,6 @@ noco.db*
/docker/main.js /docker/main.js
test_meta.db test_meta.db
test_sakila.db test_sakila.db
test_noco.db-shm
test_noco.db-wal
.env .env

206
packages/nocodb/src/lib/db/sql-client/lib/sqlite/SqliteClient.ts
Unescape View File

@ -1443,6 +1443,107 @@ class SqliteClient extends KnexClient {
return result; return result;
} }
private async _tableUpdate(args) {
const result = new Result();
args.table = args.tn;
const originalColumns = args.originalColumns;
args.connectionConfig = this._connectionConfig;
args.sqlClient = this.sqlClient;
let upQuery = '';
let downQuery = '';
for (let i = 0; i < args.columns.length; ++i) {
const oldColumn = lodash.find(originalColumns, {
cn: args.columns[i].cno,
});
if (args.columns[i].altered & 4) {
// col remove
upQuery += await this.alterTableRemoveColumn(
args.table,
args.columns[i],
oldColumn,
upQuery
);
downQuery += this.alterTableAddColumn(
args.table,
oldColumn,
args.columns[i],
downQuery
);
} else if (args.columns[i].altered & 2 || args.columns[i].altered & 8) {
// col edit
upQuery += this.alterTableChangeColumn(
args.table,
args.columns[i],
oldColumn,
upQuery
);
downQuery += ';';
// downQuery += this.alterTableChangeColumn(
// args.table,
// oldColumn,
// args.columns[i],
// downQuery,
// this.sqlClient
// );
} else if (args.columns[i].altered & 1) {
// col addition
upQuery += this.alterTableAddColumn(
args.table,
args.columns[i],
oldColumn,
upQuery
);
downQuery += ';';
// downQuery += alterTableRemoveColumn(
// args.table,
// args.columns[i],
// oldColumn,
// downQuery,
// this.sqlClient
// );
}
}
upQuery += this.alterTablePK(
args.columns,
args.originalColumns,
upQuery,
this.sqlClient
);
//downQuery += alterTablePK(args.originalColumns, args.columns, downQuery);
if (upQuery) {
//upQuery = `ALTER TABLE ${args.columns[0].tn} ${upQuery};`;
//downQuery = `ALTER TABLE ${args.columns[0].tn} ${downQuery};`;
}
await Promise.all(
upQuery.split(';').map(async (query) => {
if (query.trim().length) await this.sqlClient.raw(query);
})
);
// await this.sqlClient.raw(upQuery);
console.log(upQuery);
const afterUpdate = await this.afterTableUpdate(args);
result.data.object = {
upStatement: [
{ sql: this.querySeparator() + upQuery },
...afterUpdate.upStatement,
],
downStatement: [{ sql: ';' }],
};
return result;
}
/** /**
* *
* @param {Object} - args * @param {Object} - args
@ -1472,109 +1573,24 @@ class SqliteClient extends KnexClient {
*/ */
async tableUpdate(args) { async tableUpdate(args) {
const _func = this.tableUpdate.name; const _func = this.tableUpdate.name;
const result = new Result();
log.api(`${_func}:args:`, args); log.api(`${_func}:args:`, args);
try { for (let retry = 0; retry < 3; retry++) {
args.table = args.tn; try {
const originalColumns = args.originalColumns; return await this._tableUpdate(args);
args.connectionConfig = this._connectionConfig; } catch (e) {
args.sqlClient = this.sqlClient; console.log('retrying:tableUpdate', e);
let upQuery = '';
let downQuery = '';
for (let i = 0; i < args.columns.length; ++i) {
const oldColumn = lodash.find(originalColumns, {
cn: args.columns[i].cno,
});
if (args.columns[i].altered & 4) {
// col remove
upQuery += await this.alterTableRemoveColumn(
args.table,
args.columns[i],
oldColumn,
upQuery
);
downQuery += this.alterTableAddColumn(
args.table,
oldColumn,
args.columns[i],
downQuery
);
} else if (args.columns[i].altered & 2 || args.columns[i].altered & 8) {
// col edit
upQuery += this.alterTableChangeColumn(
args.table,
args.columns[i],
oldColumn,
upQuery
);
downQuery += ';';
// downQuery += this.alterTableChangeColumn(
// args.table,
// oldColumn,
// args.columns[i],
// downQuery,
// this.sqlClient
// );
} else if (args.columns[i].altered & 1) {
// col addition
upQuery += this.alterTableAddColumn(
args.table,
args.columns[i],
oldColumn,
upQuery
);
downQuery += ';';
// downQuery += alterTableRemoveColumn(
// args.table,
// args.columns[i],
// oldColumn,
// downQuery,
// this.sqlClient
// );
}
}
upQuery += this.alterTablePK(
args.columns,
args.originalColumns,
upQuery,
this.sqlClient
);
//downQuery += alterTablePK(args.originalColumns, args.columns, downQuery);
if (upQuery) {
//upQuery = `ALTER TABLE ${args.columns[0].tn} ${upQuery};`;
//downQuery = `ALTER TABLE ${args.columns[0].tn} ${downQuery};`;
} }
// Wait for 300ms
await new Promise((resolve) => setTimeout(resolve, 300));
}
await Promise.all( try {
upQuery.split(';').map(async (query) => { return await this._tableUpdate(args);
if (query.trim().length) await this.sqlClient.raw(query);
})
);
// await this.sqlClient.raw(upQuery);
console.log(upQuery);
const afterUpdate = await this.afterTableUpdate(args);
result.data.object = {
upStatement: [
{ sql: this.querySeparator() + upQuery },
...afterUpdate.upStatement,
],
downStatement: [{ sql: ';' }],
};
} catch (e) { } catch (e) {
log.ppe(e, _func); log.ppe(e, _func);
throw e; throw e;
} }
return result;
} }
/** /**

33
packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts
Unescape View File

@ -16,20 +16,16 @@ const sqliteSakilaSqlViews = [
]; ];
const dropTablesAndViews = async (metaKnex: Knex, prefix: string) => { const dropTablesAndViews = async (metaKnex: Knex, prefix: string) => {
for (const view of sqliteSakilaSqlViews) { try {
try { for (const view of sqliteSakilaSqlViews) {
await metaKnex.raw(`DROP VIEW IF EXISTS ${prefix}${view}`); await metaKnex.raw(`DROP VIEW IF EXISTS ${prefix}${view}`);
} catch (e) {
console.log('Error dropping sqlite view', e);
} }
}
for (const table of sakilaTableNames) { for (const table of sakilaTableNames) {
try {
await metaKnex.raw(`DROP TABLE IF EXISTS ${prefix}${table}`); await metaKnex.raw(`DROP TABLE IF EXISTS ${prefix}${table}`);
} catch (e) {
console.log('Error dropping sqlite table', e);
} }
} catch (e) {
console.error('Error dropping tables and views', e);
} }
}; };
@ -58,31 +54,34 @@ const resetMetaSakilaSqlite = async (metaKnex: Knex, prefix: string) => {
'/tests' '/tests'
); );
const trx = await metaKnex.transaction();
try { try {
const schemaFile = await fs.readFile( const schemaFile = await fs.readFile(
`${testsDir}/sqlite-sakila-db/03-sqlite-prefix-sakila-schema.sql` `${testsDir}/sqlite-sakila-db/03-sqlite-prefix-sakila-schema.sql`
); );
const schemaFileStr = schemaFile.toString().replace(/prefix___/g, prefix); const schemaFileStr = schemaFile.toString().replace(/prefix___/g, prefix);
const dataFile = await fs.readFile(
`${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql`
);
const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix);
const schemaSqlQueries = schemaFileStr.split(';'); const schemaSqlQueries = schemaFileStr.split(';');
for (const sqlQuery of schemaSqlQueries) { for (const sqlQuery of schemaSqlQueries) {
if (sqlQuery.trim().length > 0) { if (sqlQuery.trim().length > 0) {
await trx.raw( await metaKnex.raw(
sqlQuery sqlQuery
.trim() .trim()
.replace(/WHERE rowid = new.rowid/g, 'WHERE rowid = new.rowid;') .replace(/WHERE rowid = new.rowid/g, 'WHERE rowid = new.rowid;')
); );
} }
} }
} catch (e) {
console.error('Error resetting meta sakila sqlite:db', e);
}
const trx = await metaKnex.transaction();
try {
const dataFile = await fs.readFile(
`${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql`
);
const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix);
const dataSqlQueries = dataFileStr.split(';'); const dataSqlQueries = dataFileStr.split(';');
for (const sqlQuery of dataSqlQueries) { for (const sqlQuery of dataSqlQueries) {
if (sqlQuery.trim().length > 0) { if (sqlQuery.trim().length > 0) {
await trx.raw(sqlQuery.trim()); await trx.raw(sqlQuery.trim());

4
packages/nocodb/src/lib/utils/NcConfigFactory.ts
Unescape View File

@ -590,6 +590,10 @@ export default class NcConfigFactory implements NcConfig {
...args.meta.db, ...args.meta.db,
connection: args.meta.db, connection: args.meta.db,
}); });
if (process.env['TEST'] === 'true') {
await metaSqlClient.raw('PRAGMA journal_mode=WAL');
await metaSqlClient.raw('PRAGMA busy_timeout=60000');
}
await metaSqlClient.createDatabaseIfNotExists({ await metaSqlClient.createDatabaseIfNotExists({
database: args.meta.db?.connection?.filename, database: args.meta.db?.connection?.filename,
}); });

4
scripts/playwright/playwright.config.ts
Unescape View File

@ -26,9 +26,9 @@ const config: PlaywrightTestConfig = {
/* Fail the build on CI if you accidentally left test.only in the source code. */ /* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI, forbidOnly: !!process.env.CI,
/* Retry on CI only */ /* Retry on CI only */
retries: process.env.CI ? 2 : 0, retries: 2,
/* Opt out of parallel tests on CI. */ /* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : 2, workers: process.env.CI ? 2 : 4,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */ /* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'html', reporter: 'html',
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */

Write Preview
Loading…
Cancel
Save