From 770b31f99f5f334b4e552b09c02eb982ccab9986 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Mon, 17 Oct 2022 18:01:24 +0530 Subject: [PATCH] feat(testing): Improved sqlite support --- .../services/test/TestResetService/index.ts | 7 +- .../resetMetaSakilaSqliteProject.ts | 151 ++++++++++-------- .../04-sqlite-prefix-sakila-insert-data.sql | 13 -- 3 files changed, 86 insertions(+), 85 deletions(-) diff --git a/packages/nocodb/src/lib/services/test/TestResetService/index.ts b/packages/nocodb/src/lib/services/test/TestResetService/index.ts index e8994dd153..67c1adf9bd 100644 --- a/packages/nocodb/src/lib/services/test/TestResetService/index.ts +++ b/packages/nocodb/src/lib/services/test/TestResetService/index.ts @@ -76,7 +76,12 @@ export class TestResetService { } if (dbType == 'sqlite') { - await resetMetaSakilaSqliteProject({ token, metaKnex, title }); + await resetMetaSakilaSqliteProject({ + token, + metaKnex, + title, + oldProject: project, + }); } else if (dbType == 'mysql') { await resetMysqlSakilaProject({ token, diff --git a/packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts b/packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts index 85ef01e0dc..26845127b0 100644 --- a/packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts +++ b/packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts @@ -2,8 +2,8 @@ import axios from 'axios'; import Knex from 'knex'; import { promises as fs } from 'fs'; -import Audit from '../../../models/Audit'; import { sakilaTableNames } from '../../../utils/globals'; +import Project from '../../../models/Project'; const sqliteSakilaSqlViews = [ 'actor_info', @@ -15,6 +15,55 @@ const sqliteSakilaSqlViews = [ 'staff_list', ]; +const resetMetaSakilaSqliteProject = async ({ + metaKnex, + token, + title, + oldProject, +}: { + metaKnex: Knex; + token: string; + title: string; + oldProject: Project; +}) => { + const project = await createProject(token, title); + + await dropTablesAndViews(metaKnex, oldProject.prefix); + await dropTablesAndViews(metaKnex, project.prefix); + + await resetMetaSakilaSqlite(metaKnex, project.prefix, oldProject); + + await syncMeta(project, token); +}; + +const createProject = async (token: string, title: string) => { + const response = await axios.post( + 'http://localhost:8080/api/v1/db/meta/projects/', + { title }, + { + headers: { + 'xc-auth': token, + }, + } + ); + if (response.status !== 200) { + console.error('Error creating project', response.data); + } + return response.data; +}; + +const syncMeta = async (project: Project, token: string) => { + await axios.post( + `http://localhost:8080/api/v1/db/meta/projects/${project.id}/meta-diff`, + {}, + { + headers: { + 'xc-auth': token, + }, + } + ); +}; + const dropTablesAndViews = async (metaKnex: Knex, prefix: string) => { try { for (const view of sqliteSakilaSqlViews) { @@ -29,25 +78,12 @@ const dropTablesAndViews = async (metaKnex: Knex, prefix: string) => { } }; -const isMetaSakilaSqliteToBeReset = async (metaKnex: Knex, project: any) => { - const tablesInDb: Array = await metaKnex.raw( - `SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '${project.prefix}%'` - ); - - if ( - tablesInDb.length === 0 || - (tablesInDb.length > 0 && !tablesInDb.includes(`${project.prefix}actor`)) - ) { - return true; - } - - const audits = await Audit.projectAuditList(project.id, {}); - - return audits?.length > 0; -}; - -const resetMetaSakilaSqlite = async (metaKnex: Knex, prefix: string) => { - await dropTablesAndViews(metaKnex, prefix); +const resetMetaSakilaSqlite = async ( + metaKnex: Knex, + prefix: string, + oldProject: Project +) => { + await dropTablesAndViews(metaKnex, oldProject.prefix); const testsDir = __dirname.replace( '/src/lib/services/test/TestResetService', @@ -74,62 +110,35 @@ const resetMetaSakilaSqlite = async (metaKnex: Knex, prefix: string) => { console.error('Error resetting meta sakila sqlite:db', e); } - const trx = await metaKnex.transaction(); - try { - const dataFile = await fs.readFile( - `${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql` - ); - const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix); - const dataSqlQueries = dataFileStr.split(';'); - - for (const sqlQuery of dataSqlQueries) { - if (sqlQuery.trim().length > 0) { - await trx.raw(sqlQuery.trim()); - } + const dataFile = await fs.readFile( + `${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql` + ); + const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix); + const dataSqlQueries = dataFileStr + .split(';') + .filter((str) => str.trim().length > 0) + .map((str) => str.trim()); + + const batchSize = 1000; + const batches = dataSqlQueries.reduce((acc, _, i) => { + if (!(i % batchSize)) { + // if index is 0 or can be divided by the `size`... + acc.push(dataSqlQueries.slice(i, i + batchSize)); // ..push a chunk of the original array to the accumulator } - await trx.commit(); - } catch (e) { - console.log('Error resetting sqlite db', e); - await trx.rollback(e); - } -}; + return acc; + }, []); -const resetMetaSakilaSqliteProject = async ({ - metaKnex, - token, - title, -}: { - metaKnex: Knex; - token: string; - title: string; -}) => { - const response = await axios.post( - 'http://localhost:8080/api/v1/db/meta/projects/', - { title }, - { - headers: { - 'xc-auth': token, - }, + for (const sqlQueryBatch of batches) { + const trx = await metaKnex.transaction(); + + for (const sqlQuery of sqlQueryBatch) { + await trx.raw(sqlQuery); } - ); - if (response.status !== 200) { - console.error('Error creating project', response.data); - } - const project = response.data; - if (await isMetaSakilaSqliteToBeReset(metaKnex, project)) { - await resetMetaSakilaSqlite(metaKnex, project.prefix); + await trx.commit(); + // wait for 40 ms to avoid SQLITE_BUSY error + await new Promise((resolve) => setTimeout(resolve, 40)); } - - await axios.post( - `http://localhost:8080/api/v1/db/meta/projects/${project.id}/meta-diff`, - {}, - { - headers: { - 'xc-auth': token, - }, - } - ); }; export default resetMetaSakilaSqliteProject; diff --git a/packages/nocodb/tests/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql b/packages/nocodb/tests/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql index 8a894f3ea0..4fdfcf07be 100644 --- a/packages/nocodb/tests/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql +++ b/packages/nocodb/tests/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql @@ -1,17 +1,4 @@ -/* -Sakila for SQLite is a port of the Sakila example database available for MySQL, which was originally developed by Mike Hillyer of the MySQL AB documentation team. -This project is designed to help database administrators to decide which database to use for development of new products -The user can run the same SQL against different kind of databases and compare the performance - -License: BSD -Copyright DB Software Laboratory -http://www.etl-tools.com - -*/ - - --- Delete data DELETE FROM prefix___payment ; DELETE FROM prefix___rental