Browse Source

feat: migrate to better-sqlite3

Signed-off-by: mertmit <mertmit99@gmail.com>
feat/better-sqlite3
mertmit 2 years ago
parent
commit
83a7ad1c05
  1. 2
      packages/nc-gui/lib/enums.ts
  2. 2
      packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts
  3. 14651
      packages/nocodb/package-lock.json
  4. 3
      packages/nocodb/package.json
  5. 5
      packages/nocodb/src/lib/db/sql-client/lib/KnexClient.ts
  6. 4
      packages/nocodb/src/lib/db/sql-client/lib/SqlClientFactory.ts
  7. 3
      packages/nocodb/src/lib/db/sql-client/lib/sqlite/SqliteClient.ts
  8. 2
      packages/nocodb/src/lib/db/sql-data-mapper/__tests__/conditionClause.test.js
  9. 2
      packages/nocodb/src/lib/db/sql-data-mapper/lib/BaseModel.ts
  10. 3
      packages/nocodb/src/lib/db/sql-data-mapper/lib/DbFactory.ts
  11. 2
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSql.ts
  12. 2
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts
  13. 4
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/CustomKnex.ts
  14. 2
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulaQueryBuilderFromString.ts
  15. 4
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts
  16. 1
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/mapFunctionName.ts
  17. 10
      packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts
  18. 2
      packages/nocodb/src/lib/db/sql-mgr/code/gql-schema/xc-ts/GqlXcSchemaFactory.ts
  19. 2
      packages/nocodb/src/lib/db/sql-mgr/code/models/xc/ModelXcMetaFactory.ts
  20. 1
      packages/nocodb/src/lib/db/sql-mgr/code/routers/xc-ts/SwaggerTypes.ts
  21. 4
      packages/nocodb/src/lib/db/sql-migrator/lib/KnexMigrator.ts
  22. 4
      packages/nocodb/src/lib/db/sql-migrator/lib/KnexMigratorv2.ts
  23. 1
      packages/nocodb/src/lib/db/sql-migrator/lib/SqlMigratorFactory.ts
  24. 8
      packages/nocodb/src/lib/db/sql-migrator/lib/templates/sqlite.template.ts
  25. 2
      packages/nocodb/src/lib/meta/MetaAPILogger.ts
  26. 2
      packages/nocodb/src/lib/meta/NcMetaIOImpl.ts
  27. 10
      packages/nocodb/src/lib/meta/api/columnApis.ts
  28. 6
      packages/nocodb/src/lib/meta/api/projectApis.ts
  29. 198
      packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts
  30. 87
      packages/nocodb/src/lib/meta/api/sync/helpers/job.ts
  31. 7
      packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts
  32. 4
      packages/nocodb/src/lib/migrations/v2/nc_012_alter_column_data_types.ts
  33. 4
      packages/nocodb/src/lib/migrations/v2/nc_014_alter_column_data_types.ts
  34. 4
      packages/nocodb/src/lib/migrations/v2/nc_016_alter_hooklog_payload_types.ts
  35. 4
      packages/nocodb/src/lib/models/Base.ts
  36. 26
      packages/nocodb/src/lib/utils/NcConfigFactory.ts
  37. 2
      packages/nocodb/src/lib/utils/common/NcConnectionMgr.ts
  38. 4
      packages/nocodb/src/lib/v1-legacy/templates/NcTemplateParser.ts
  39. 4
      packages/nocodb/tests/unit/TestDbMngr.ts
  40. 2
      packages/nocodb/tests/unit/init/db.ts

2
packages/nc-gui/lib/enums.ts

@ -17,7 +17,7 @@ export enum ClientType {
MYSQL = 'mysql2', MYSQL = 'mysql2',
MSSQL = 'mssql', MSSQL = 'mssql',
PG = 'pg', PG = 'pg',
SQLITE = 'sqlite3', SQLITE = 'better-sqlite3',
VITESS = 'vitess', VITESS = 'vitess',
} }

2
packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts

@ -26,7 +26,7 @@ export class SqlUiFactory {
return MysqlUi; return MysqlUi;
} }
if (connectionConfig.client === 'sqlite3') { if (connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3') {
return SqliteUi; return SqliteUi;
} }
if (connectionConfig.client === 'mssql') { if (connectionConfig.client === 'mssql') {

14651
packages/nocodb/package-lock.json generated

File diff suppressed because it is too large Load Diff

3
packages/nocodb/package.json

@ -62,6 +62,7 @@
"aws-sdk": "^2.829.0", "aws-sdk": "^2.829.0",
"axios": "^0.21.1", "axios": "^0.21.1",
"bcryptjs": "^2.4.3", "bcryptjs": "^2.4.3",
"better-sqlite3": "^7.6.2",
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"boxen": "^5.0.0", "boxen": "^5.0.0",
"bullmq": "^1.81.1", "bullmq": "^1.81.1",
@ -130,7 +131,6 @@
"rmdir": "^1.2.0", "rmdir": "^1.2.0",
"slash": "^3.0.0", "slash": "^3.0.0",
"socket.io": "^4.4.1", "socket.io": "^4.4.1",
"sqlite3": "5.1.2",
"tedious": "^15.0.0", "tedious": "^15.0.0",
"tinycolor2": "^1.4.2", "tinycolor2": "^1.4.2",
"twilio": "^3.55.1", "twilio": "^3.55.1",
@ -141,6 +141,7 @@
"xlsx": "^0.18.5" "xlsx": "^0.18.5"
}, },
"devDependencies": { "devDependencies": {
"@types/better-sqlite3": "^7.6.2",
"@types/chai": "^4.2.12", "@types/chai": "^4.2.12",
"@types/express": "^4.17.7", "@types/express": "^4.17.7",
"@types/inflection": "^1.5.28", "@types/inflection": "^1.5.28",

5
packages/nocodb/src/lib/db/sql-client/lib/KnexClient.ts

@ -608,9 +608,12 @@ class KnexClient extends SqlClient {
} }
} }
const tmpConnectionConfig = const tmpConnectionConfig =
connectionConfig.client === 'sqlite3' connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3'
? connectionConfig.connection ? connectionConfig.connection
: connectionConfig; : connectionConfig;
tmpConnectionConfig.client = tmpConnectionConfig.client === 'sqlite3' ? 'better-sqlite3' : tmpConnectionConfig.client;
this.sqlClient = knex(tmpConnectionConfig); this.sqlClient = knex(tmpConnectionConfig);
} }
this.knex = this.sqlClient; this.knex = this.sqlClient;

4
packages/nocodb/src/lib/db/sql-client/lib/SqlClientFactory.ts

@ -21,7 +21,9 @@ class SqlClientFactory {
if (connectionConfig.meta.dbtype === 'vitess') if (connectionConfig.meta.dbtype === 'vitess')
return new VitessClient(connectionConfig); return new VitessClient(connectionConfig);
return new MySqlClient(connectionConfig); return new MySqlClient(connectionConfig);
} else if (connectionConfig.client === 'sqlite3') { } else if (connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3') {
connectionConfig.client = 'better-sqlite3';
connectionConfig.connection.client = 'better-sqlite3';
return new SqliteClient(connectionConfig); return new SqliteClient(connectionConfig);
} else if (connectionConfig.client === 'mssql') { } else if (connectionConfig.client === 'mssql') {
return new MssqlClient(connectionConfig); return new MssqlClient(connectionConfig);

3
packages/nocodb/src/lib/db/sql-client/lib/sqlite/SqliteClient.ts

@ -16,6 +16,7 @@ class SqliteClient extends KnexClient {
constructor(connectionConfig) { constructor(connectionConfig) {
super(connectionConfig); super(connectionConfig);
connectionConfig.connection.client = connectionConfig.connection.client === 'sqlite3' ? 'better-sqlite3' : connectionConfig.connection.client;
this.sqlClient = knex(connectionConfig.connection); this.sqlClient = knex(connectionConfig.connection);
this.queries = queries; this.queries = queries;
this._version = {}; this._version = {};
@ -127,6 +128,8 @@ class SqliteClient extends KnexClient {
try { try {
const exists = await promisify(fs.exists)(args.database); const exists = await promisify(fs.exists)(args.database);
this.connectionConfig.connection.client = this.connectionConfig.connection.client === 'sqlite3' ? 'better-sqlite3' : this.connectionConfig.connection.client;
if (!exists) { if (!exists) {
log.debug('sqlite file do no exists - create one'); log.debug('sqlite file do no exists - create one');
const fd = await promisify(fs.open)(args.database, 'w'); const fd = await promisify(fs.open)(args.database, 'w');

2
packages/nocodb/src/lib/db/sql-data-mapper/__tests__/conditionClause.test.js

@ -100,7 +100,7 @@ it(`Test`, function (done) {
}, },
{ {
knex :cstomKnex({ knex :cstomKnex({
client: 'sqlite3' client: 'better-sqlite3'
}), }),
expectedOp: 'select * from `test` inner join `test1` on `test1`.`test_ref_id` = `test`.`test_id` inner join `test3` on `test1`.`test3_ref_id` = `test3`.`test3_id` where `test3`.`test3Col` = \'123\'' expectedOp: 'select * from `test` inner join `test1` on `test1`.`test_ref_id` = `test`.`test_id` inner join `test3` on `test1`.`test3_ref_id` = `test3`.`test3_id` where `test3`.`test3Col` = \'123\''
}, },

2
packages/nocodb/src/lib/db/sql-data-mapper/lib/BaseModel.ts

@ -1193,7 +1193,7 @@ abstract class BaseModel {
} }
isSqlite(): boolean { isSqlite(): boolean {
return this.clientType === 'sqlite3'; return this.clientType === 'sqlite3' || this.clientType === 'better-sqlite3';
} }
/** /**

3
packages/nocodb/src/lib/db/sql-data-mapper/lib/DbFactory.ts

@ -2,7 +2,8 @@ import knex from './sql/CustomKnex';
export class DbFactory { export class DbFactory {
static create(connectionConfig) { static create(connectionConfig) {
if (connectionConfig.client === 'sqlite3') { if (connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3') {
connectionConfig.connection.client = connectionConfig.connection.client === 'sqlite3' ? 'better-sqlite3' : connectionConfig.connection.client;
return knex(connectionConfig.connection); return knex(connectionConfig.connection);
} else if ( } else if (
['mysql', 'mysql2', 'pg', 'mssql'].includes(connectionConfig.client) ['mysql', 'mysql2', 'pg', 'mssql'].includes(connectionConfig.client)

2
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSql.ts

@ -2095,7 +2095,7 @@ class BaseModelSql extends BaseModel {
} }
isSqlite() { isSqlite() {
return this.clientType === 'sqlite3'; return this.clientType === 'sqlite3' || this.clientType === 'better-sqlite3';
} }
isMssql() { isMssql() {

2
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@ -1639,7 +1639,7 @@ class BaseModelSqlv2 {
} }
get isSqlite() { get isSqlite() {
return this.clientType === 'sqlite3'; return this.clientType === 'sqlite3' || this.clientType === 'better-sqlite3';
} }
get isMssql() { get isMssql() {

4
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/CustomKnex.ts

@ -557,6 +557,7 @@ knex.QueryBuilder.extend('concat', function (cn: any) {
this.select(this.client.raw(`STRING_AGG(??, ',')`, [cn])); this.select(this.client.raw(`STRING_AGG(??, ',')`, [cn]));
break; break;
case 'sqlite3': case 'sqlite3':
case 'better-sqlite3':
this.select(this.client.raw(`GROUP_CONCAT(?? , ',')`, [cn])); this.select(this.client.raw(`GROUP_CONCAT(?? , ',')`, [cn]));
break; break;
} }
@ -988,6 +989,9 @@ function parseNestedCondition(obj, qb, pKey?, table?, tableAlias?) {
type CustomKnex = Knex; type CustomKnex = Knex;
function CustomKnex(arg: string | Knex.Config<any> | any): CustomKnex { function CustomKnex(arg: string | Knex.Config<any> | any): CustomKnex {
if (arg?.client === 'sqlite3') {
arg.client = 'better-sqlite3';
}
const kn: any = knex(arg); const kn: any = knex(arg);
const knexRaw = kn.raw; const knexRaw = kn.raw;

2
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulaQueryBuilderFromString.ts

@ -47,7 +47,7 @@ export default function formulaQueryBuilder(
// } // }
// break; // break;
case 'CONCAT': case 'CONCAT':
if (knex.clientType() === 'sqlite3') { if (knex.clientType() === 'better-sqlite3') {
if (pt.arguments.length > 1) { if (pt.arguments.length > 1) {
return fn( return fn(
{ {

4
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts

@ -549,7 +549,7 @@ export default async function formulaQueryBuilderv2(
// } // }
// break; // break;
case 'CONCAT': case 'CONCAT':
if (knex.clientType() === 'sqlite3') { if (knex.clientType() === 'better-sqlite3') {
if (pt.arguments.length > 1) { if (pt.arguments.length > 1) {
return fn( return fn(
{ {
@ -655,7 +655,7 @@ export default async function formulaQueryBuilderv2(
let sql = `${left} ${pt.operator} ${right}${colAlias}`; let sql = `${left} ${pt.operator} ${right}${colAlias}`;
// handle NULL values when calling CONCAT for sqlite3 // handle NULL values when calling CONCAT for sqlite3
if (pt.left.fnName === 'CONCAT' && knex.clientType() === 'sqlite3') { if (pt.left.fnName === 'CONCAT' && knex.clientType() === 'better-sqlite3') {
sql = `COALESCE(${left}, '') ${pt.operator} COALESCE(${right},'')${colAlias}`; sql = `COALESCE(${left}, '') ${pt.operator} COALESCE(${right},'')${colAlias}`;
} }

1
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/mapFunctionName.ts

@ -35,6 +35,7 @@ const mapFunctionName = (args: MapFnArgs): any => {
break; break;
case 'sqlite': case 'sqlite':
case 'sqlite3': case 'sqlite3':
case 'better-sqlite3':
val = sqlite[name] || name; val = sqlite[name] || name;
break; break;
} }

10
packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts

@ -386,7 +386,7 @@ export default class SqlMgr {
} }
} }
if (connectionConfig.client === 'sqlite3') { if (connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3') {
schemaKey = connectionConfig.connection.connection.filename; schemaKey = connectionConfig.connection.connection.filename;
} else { } else {
schemaKey = schemaKey =
@ -503,7 +503,8 @@ export default class SqlMgr {
return 'mssql'; return 'mssql';
break; break;
case 'sqlite3:': case 'sqlite3:':
return 'sqlite3'; case 'better-sqlite3:':
return 'better-sqlite3';
break; break;
case 'cockroachdb:': case 'cockroachdb:':
return 'pg'; return 'pg';
@ -518,9 +519,9 @@ export default class SqlMgr {
const ORACLE_PORT = 1521; const ORACLE_PORT = 1521;
if (sqlConfig.typeOfDatabase === 'sqlite3') { if (sqlConfig.typeOfDatabase === 'sqlite3' || sqlConfig.typeOfDatabase === 'better-sqlite3') {
return { return {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
// filename: "./db/sakila-sqlite" // filename: "./db/sakila-sqlite"
filename: sqlConfig.database, filename: sqlConfig.database,
@ -575,6 +576,7 @@ export default class SqlMgr {
return 1433; return 1433;
break; break;
case 'sqlite3': case 'sqlite3':
case 'better-sqlite3':
return 0; return 0;
break; break;
default: default:

2
packages/nocodb/src/lib/db/sql-mgr/code/gql-schema/xc-ts/GqlXcSchemaFactory.ts

@ -11,7 +11,7 @@ class GqlXcSchemaFactory {
connectionConfig.client === 'mysql' connectionConfig.client === 'mysql'
) { ) {
return new GqlXcTsSchemaMysql(args); return new GqlXcTsSchemaMysql(args);
} else if (connectionConfig.client === 'sqlite3') { } else if (connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3') {
return new GqlXcSchemaSqlite(args); return new GqlXcSchemaSqlite(args);
} else if (connectionConfig.client === 'mssql') { } else if (connectionConfig.client === 'mssql') {
return new GqlXcSchemaMssql(args); return new GqlXcSchemaMssql(args);

2
packages/nocodb/src/lib/db/sql-mgr/code/models/xc/ModelXcMetaFactory.ts

@ -12,7 +12,7 @@ class ModelXcMetaFactory {
connectionConfig.client === 'mysql' connectionConfig.client === 'mysql'
) { ) {
return new ModelXcMetaMysql(args); return new ModelXcMetaMysql(args);
} else if (connectionConfig.client === 'sqlite3') { } else if (connectionConfig.client === 'sqlite3' || connectionConfig.client === 'better-sqlite3') {
return new ModelXcMetaSqlite(args); return new ModelXcMetaSqlite(args);
} else if (connectionConfig.client === 'mssql') { } else if (connectionConfig.client === 'mssql') {
return new ModelXcMetaMssql(args); return new ModelXcMetaMssql(args);

1
packages/nocodb/src/lib/db/sql-mgr/code/routers/xc-ts/SwaggerTypes.ts

@ -13,6 +13,7 @@ class SwaggerTypes {
SwaggerTypes.setSwaggerTypeForMssql(column, field); SwaggerTypes.setSwaggerTypeForMssql(column, field);
break; break;
case 'sqlite3': case 'sqlite3':
case 'better-sqlite3':
SwaggerTypes.setSwaggerTypeForSqlite(column, field); SwaggerTypes.setSwaggerTypeForSqlite(column, field);
break; break;
} }

4
packages/nocodb/src/lib/db/sql-migrator/lib/KnexMigrator.ts

@ -389,7 +389,7 @@ export default class KnexMigrator extends SqlMigrator {
await sqlClient.createDatabaseIfNotExists({ await sqlClient.createDatabaseIfNotExists({
database: connectionConfig.connection.user, database: connectionConfig.connection.user,
}); });
} else if (connectionConfig.client !== 'sqlite3') { } else if (connectionConfig.client !== 'sqlite3' && connectionConfig.client !== 'better-sqlite3') {
this.emit( this.emit(
`${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}` `${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}`
); );
@ -433,7 +433,7 @@ export default class KnexMigrator extends SqlMigrator {
await sqlClient.dropDatabase({ await sqlClient.dropDatabase({
database: connectionConfig.connection.user, database: connectionConfig.connection.user,
}); });
} else if (connectionConfig.client === 'sqlite3') { } else if (connectionConfig.client !== 'sqlite3' && connectionConfig.client === 'better-sqlite3') {
this.emit( this.emit(
`Dropping DB : ${connectionConfig.connection.connection.filename}` `Dropping DB : ${connectionConfig.connection.connection.filename}`
); );

4
packages/nocodb/src/lib/db/sql-migrator/lib/KnexMigratorv2.ts

@ -391,7 +391,7 @@ export default class KnexMigratorv2 {
await sqlClient.createDatabaseIfNotExists({ await sqlClient.createDatabaseIfNotExists({
database: connectionConfig.connection.user, database: connectionConfig.connection.user,
}); });
} else if (connectionConfig.client !== 'sqlite3') { } else if (connectionConfig.client !== 'sqlite3' && connectionConfig.client !== 'better-sqlite3') {
this.emit( this.emit(
`${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}` `${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}`
); );
@ -439,7 +439,7 @@ export default class KnexMigratorv2 {
await sqlClient.dropDatabase({ await sqlClient.dropDatabase({
database: connectionConfig.connection.user, database: connectionConfig.connection.user,
}); });
} else if (connectionConfig.client === 'sqlite3') { } else if (connectionConfig.client !== 'sqlite3' && connectionConfig.client === 'better-sqlite3') {
this.emit( this.emit(
`Dropping DB : ${connectionConfig.connection.connection.filename}` `Dropping DB : ${connectionConfig.connection.connection.filename}`
); );

1
packages/nocodb/src/lib/db/sql-migrator/lib/SqlMigratorFactory.ts

@ -9,6 +9,7 @@ export default class SqlMigratorFactory {
case 'oracledb': case 'oracledb':
case 'mssql': case 'mssql':
case 'sqlite3': case 'sqlite3':
case 'better-sqlite3':
return new KnexMigrator(); return new KnexMigrator();
break; break;
default: default:

8
packages/nocodb/src/lib/db/sql-migrator/lib/templates/sqlite.template.ts

@ -8,9 +8,9 @@ module.exports = {
_noco: { _noco: {
db: [ db: [
{ {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: filename:
DOCKER_DB_FILE || DOCKER_DB_FILE ||
@ -29,9 +29,9 @@ module.exports = {
api: {}, api: {},
db: [ db: [
{ {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: filename:
DOCKER_DB_FILE || DOCKER_DB_FILE ||

2
packages/nocodb/src/lib/meta/MetaAPILogger.ts

@ -7,7 +7,7 @@ export default class MetaAPILogger {
constructor() { constructor() {
this.knex = XKnex({ this.knex = XKnex({
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: 'noco_log.db', filename: 'noco_log.db',
}, },

2
packages/nocodb/src/lib/meta/NcMetaIOImpl.ts

@ -77,7 +77,7 @@ export default class NcMetaIOImpl extends NcMetaIO {
constructor(app: Noco, config: NcConfig, trx = null) { constructor(app: Noco, config: NcConfig, trx = null) {
super(app, config); super(app, config);
if (this.config?.meta?.db?.client === 'sqlite3') { if (this.config?.meta?.db?.client === 'sqlite3' || this.config?.meta?.db?.client === 'better-sqlite3') {
this.config.meta.db.useNullAsDefault = true; this.config.meta.db.useNullAsDefault = true;
} }

10
packages/nocodb/src/lib/meta/api/columnApis.ts

@ -762,7 +762,7 @@ export async function columnUpdate(req: Request, res: Response<TableType>) {
}); });
if (colBody.colOptions?.options) { if (colBody.colOptions?.options) {
const supportedDrivers = ['mysql', 'mysql2', 'pg', 'mssql', 'sqlite3']; const supportedDrivers = ['mysql', 'mysql2', 'pg', 'mssql', 'sqlite3', 'better-sqlite3'];
const dbDriver = NcConnectionMgrv2.get(base); const dbDriver = NcConnectionMgrv2.get(base);
const driverType = dbDriver.clientType(); const driverType = dbDriver.clientType();
@ -798,7 +798,7 @@ export async function columnUpdate(req: Request, res: Response<TableType>) {
column.column_name, column.column_name,
] ]
); );
} else if (driverType === 'sqlite3') { } else if (driverType === 'sqlite3' || driverType === 'better-sqlite3') {
await dbDriver.raw( await dbDriver.raw(
`UPDATE ?? SET ?? = substr(??, 1, instr(??, ',') - 1) WHERE ?? LIKE '%,%';`, `UPDATE ?? SET ?? = substr(??, 1, instr(??, ',') - 1) WHERE ?? LIKE '%,%';`,
[ [
@ -992,7 +992,7 @@ export async function columnUpdate(req: Request, res: Response<TableType>) {
option.title, option.title,
] ]
); );
} else if (driverType === 'sqlite3') { } else if (driverType === 'sqlite3' || driverType === 'better-sqlite3') {
await dbDriver.raw( await dbDriver.raw(
`UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ','), ',')`, `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ','), ',')`,
[ [
@ -1175,7 +1175,7 @@ export async function columnUpdate(req: Request, res: Response<TableType>) {
newOp.title, newOp.title,
] ]
); );
} else if (driverType === 'sqlite3') { } else if (driverType === 'sqlite3' || driverType === 'better-sqlite3') {
await dbDriver.raw( await dbDriver.raw(
`UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`, `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`,
[ [
@ -1263,7 +1263,7 @@ export async function columnUpdate(req: Request, res: Response<TableType>) {
newOp.title, newOp.title,
] ]
); );
} else if (driverType === 'sqlite3') { } else if (driverType === 'sqlite3' || driverType === 'better-sqlite3') {
await dbDriver.raw( await dbDriver.raw(
`UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`, `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`,
[ [

6
packages/nocodb/src/lib/meta/api/projectApis.ts

@ -125,11 +125,11 @@ async function projectCreate(req: Request<any, any>, res) {
projectBody.prefix = ''; projectBody.prefix = '';
projectBody.bases = [ projectBody.bases = [
{ {
type: 'sqlite3', type: 'better-sqlite3',
config: { config: {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
client: 'sqlite3', client: 'better-sqlite3',
database: projectTitle, database: projectTitle,
connection: { connection: {
filename: `${toolDir}/nc_minimal_dbs/${projectTitle}_${dbId}.db`, filename: `${toolDir}/nc_minimal_dbs/${projectTitle}_${dbId}.db`,

198
packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts

@ -1,139 +1,90 @@
import sqlite3 from 'sqlite3'; import sqlite3 from 'better-sqlite3';
import { Readable } from 'stream'; import { Readable } from 'stream';
class EntityMap { class EntityMap {
initialized: boolean;
cols: string[]; cols: string[];
db: any; db: any;
constructor(...args) { constructor(...args) {
this.initialized = false;
this.cols = args.map((arg) => processKey(arg)); this.cols = args.map((arg) => processKey(arg));
this.db = new Promise((resolve, reject) => { this.db = new sqlite3(':memory:');
const db = new sqlite3.Database(':memory:');
const colStatement =
const colStatement = this.cols.length > 0
this.cols.length > 0 ? this.cols.join(' TEXT, ') + ' TEXT'
? this.cols.join(' TEXT, ') + ' TEXT' : 'mappingPlaceholder TEXT';
: 'mappingPlaceholder TEXT'; const stmt = this.db.prepare(`CREATE TABLE mapping (${colStatement})`);
db.run(`CREATE TABLE mapping (${colStatement})`, (err) => { stmt.run();
if (err) {
console.log(err);
reject(err);
}
resolve(db);
});
});
}
async init() {
if (!this.initialized) {
this.db = await this.db;
this.initialized = true;
}
} }
destroy() { destroy() {
if (this.initialized && this.db) { if (this.db) {
this.db.close(); this.db.close();
} }
} }
async addRow(row) { addRow(row): void {
if (!this.initialized) {
throw 'Please initialize first!';
}
const cols = Object.keys(row).map((key) => processKey(key)); const cols = Object.keys(row).map((key) => processKey(key));
const colStatement = cols.map((key) => `'${key}'`).join(', '); const colStatement = cols.map((key) => `'${key}'`).join(', ');
const questionMarks = cols.map(() => '?').join(', '); const questionMarks = cols.map(() => '?').join(', ');
const promises = [];
for (const col of cols.filter((col) => !this.cols.includes(col))) { for (const col of cols.filter((col) => !this.cols.includes(col))) {
promises.push( try {
new Promise((resolve, reject) => { const stmt = this.db.prepare(`ALTER TABLE mapping ADD '${col}' TEXT;`);
this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => { stmt.run();
if (err) { this.cols.push(col);
console.log(err); } catch (e) {
reject(err); console.log(e);
} }
this.cols.push(col);
resolve(true);
});
})
);
} }
await Promise.all(promises);
const values = Object.values(row).map((val) => { const values = Object.values(row).map((val) => {
if (typeof val === 'object') { if (typeof val === 'object' || typeof val === 'boolean') {
return `JSON::${JSON.stringify(val)}`; return `JSON::${JSON.stringify(val)}`;
} }
return val; return `${val}`;
}); });
return new Promise((resolve, reject) => { try {
this.db.run( const stmt = this.db.prepare(
`INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`, `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`
values,
(err) => {
if (err) {
console.log(err);
reject(err);
}
resolve(true);
}
); );
}); stmt.run(values);
} catch (e) {
console.log(colStatement);
console.log(values);
console.log(e);
}
} }
getRow(col, val, res = []): Promise<Record<string, any>> { getRow(col, val, res = []): Record<string, any> {
if (!this.initialized) { col = processKey(col);
throw 'Please initialize first!'; res = res.map((r) => processKey(r));
}
return new Promise((resolve, reject) => { try {
col = processKey(col); const stmt = this.db.prepare(
res = res.map((r) => processKey(r)); `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping WHERE ${col} = ?`
this.db.get(
`SELECT ${
res.length ? res.join(', ') : '*'
} FROM mapping WHERE ${col} = ?`,
[val],
(err, rs) => {
if (err) {
console.log(err);
reject(err);
}
if (rs) {
rs = processResponseRow(rs);
}
resolve(rs);
}
); );
}); const row = stmt.get(val);
return processResponseRow(row);
} catch (e) {
console.log(e);
return null;
}
} }
getCount(): Promise<number> { getCount(): number {
if (!this.initialized) { try {
throw 'Please initialize first!'; const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM mapping`);
const row = stmt.get();
return row.count;
} catch (e) {
console.log(e);
return 0;
} }
return new Promise((resolve, reject) => {
this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => {
if (err) {
console.log(err);
reject(err);
}
resolve(rs.count);
});
});
} }
getStream(res = []): DBStream { getStream(res = []): DBStream {
if (!this.initialized) {
throw 'Please initialize first!';
}
res = res.map((r) => processKey(r)); res = res.map((r) => processKey(r));
return new DBStream( return new DBStream(
this.db, this.db,
@ -141,28 +92,17 @@ class EntityMap {
); );
} }
getLimit(limit, offset, res = []): Promise<Record<string, any>[]> { getLimit(limit, offset, res = []): Record<string, any>[] {
if (!this.initialized) { try {
throw 'Please initialize first!'; const stmt = this.db.prepare(
} `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping LIMIT ${limit} OFFSET ${offset}`
return new Promise((resolve, reject) => {
res = res.map((r) => processKey(r));
this.db.all(
`SELECT ${
res.length ? res.join(', ') : '*'
} FROM mapping LIMIT ${limit} OFFSET ${offset}`,
(err, rs) => {
if (err) {
console.log(err);
reject(err);
}
for (let row of rs) {
row = processResponseRow(row);
}
resolve(rs);
}
); );
}); const rows = stmt.all();
return rows.map(processResponseRow);
} catch (e) {
console.log(e);
return [];
}
} }
} }
@ -176,25 +116,19 @@ class DBStream extends Readable {
this.db = db; this.db = db;
this.sql = sql; this.sql = sql;
this.stmt = this.db.prepare(this.sql); this.stmt = this.db.prepare(this.sql);
this.on('end', () => this.stmt.finalize());
} }
_read() { _read() {
let stream = this; let stream = this;
this.stmt.get(function (err, result) { for (const row of this.stmt.iterate()) {
if (err) { stream.push(processResponseRow(row));
stream.emit('error', err); }
} else { stream.push(null);
if (result) {
result = processResponseRow(result);
}
stream.push(result || null);
}
});
} }
} }
function processResponseRow(res: any) { function processResponseRow(res?: any) {
if (!res) return null;
for (const key of Object.keys(res)) { for (const key of Object.keys(res)) {
if (res[key] && res[key].startsWith('JSON::')) { if (res[key] && res[key].startsWith('JSON::')) {
try { try {

87
packages/nocodb/src/lib/meta/api/sync/helpers/job.ts

@ -70,27 +70,26 @@ export default async (
progress: (data: { msg?: string; level?: any }) => void progress: (data: { msg?: string; level?: any }) => void
) => { ) => {
const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
await sMapEM.init();
const sMap = { const sMap = {
// static mapping records between aTblId && ncId // static mapping records between aTblId && ncId
async addToMappingTbl(aTblId, ncId, ncName, ncParent?) { addToMappingTbl(aTblId, ncId, ncName, ncParent?) {
await sMapEM.addRow({ aTblId, ncId, ncName, ncParent }); sMapEM.addRow({ aTblId, ncId, ncName, ncParent });
}, },
// get NcID from airtable ID // get NcID from airtable ID
async getNcIdFromAtId(aId) { getNcIdFromAtId(aId) {
return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId; return sMapEM.getRow('aTblId', aId, ['ncId'])?.ncId;
}, },
// get nc Parent from airtable ID // get nc Parent from airtable ID
async getNcParentFromAtId(aId) { getNcParentFromAtId(aId) {
return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent; return sMapEM.getRow('aTblId', aId, ['ncParent'])?.ncParent;
}, },
// get nc-title from airtable ID // get nc-title from airtable ID
async getNcNameFromAtId(aId) { getNcNameFromAtId(aId) {
return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName; return sMapEM.getRow('aTblId', aId, ['ncName'])?.ncName;
}, },
}; };
@ -331,8 +330,8 @@ export default async (
// let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn);
// return ncCol; // return ncCol;
const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId); const ncTblId = sMap.getNcParentFromAtId(aTblFieldId);
const ncColId = await sMap.getNcIdFromAtId(aTblFieldId); const ncColId = sMap.getNcIdFromAtId(aTblFieldId);
// not migrated column, skip // not migrated column, skip
if (ncColId === undefined || ncTblId === undefined) return 0; if (ncColId === undefined || ncTblId === undefined) return 0;
@ -460,7 +459,7 @@ export default async (
: tinycolor.random().toHexString(), : tinycolor.random().toHexString(),
}); });
await sMap.addToMappingTbl( sMap.addToMappingTbl(
(value as any).id, (value as any).id,
undefined, undefined,
(value as any).name (value as any).name
@ -563,7 +562,7 @@ export default async (
if (col.type === 'text') ncCol.dt = 'text'; if (col.type === 'text') ncCol.dt = 'text';
// #fix-2363-decimal-out-of-range // #fix-2363-decimal-out-of-range
if (['sqlite3', 'mysql2'].includes(getRootDbType())) { if (['sqlite3', 'better-sqlite3', 'mysql2'].includes(getRootDbType())) {
if (ncCol.uidt === UITypes.Decimal) { if (ncCol.uidt === UITypes.Decimal) {
ncCol.dt = 'double'; ncCol.dt = 'double';
ncCol.dtxp = 22; ncCol.dtxp = 22;
@ -623,14 +622,14 @@ export default async (
updateNcTblSchema(table); updateNcTblSchema(table);
// update mapping table // update mapping table
await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title);
for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { for (let colIdx = 0; colIdx < table.columns.length; colIdx++) {
const aId = aTblSchema[idx].columns.find( const aId = aTblSchema[idx].columns.find(
(x) => (x) =>
x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title
)?.id; )?.id;
if (aId) if (aId)
await sMap.addToMappingTbl( sMap.addToMappingTbl(
aId, aId,
table.columns[colIdx].id, table.columns[colIdx].id,
table.columns[colIdx].title, table.columns[colIdx].title,
@ -654,7 +653,7 @@ export default async (
await updateNcTblSchemaById(table.id); await updateNcTblSchemaById(table.id);
await sMap.addToMappingTbl( sMap.addToMappingTbl(
aTbl_grid.id, aTbl_grid.id,
table.views[0].id, table.views[0].id,
aTbl_grid.name, aTbl_grid.name,
@ -699,7 +698,7 @@ export default async (
if (!nc_isLinkExists(aTblLinkColumns[i].id)) { if (!nc_isLinkExists(aTblLinkColumns[i].id)) {
// parent table ID // parent table ID
// let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
// find child table name from symmetric column ID specified // find child table name from symmetric column ID specified
// self link, symmetricColumnId field will be undefined // self link, symmetricColumnId field will be undefined
@ -750,7 +749,7 @@ export default async (
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
(x) => x.title === ncName.title (x) => x.title === ncName.title
)?.id; )?.id;
await sMap.addToMappingTbl( sMap.addToMappingTbl(
aTblLinkColumns[i].id, aTblLinkColumns[i].id,
ncId, ncId,
ncName.title, ncName.title,
@ -891,7 +890,7 @@ export default async (
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
(x) => x.title === aTblLinkColumns[i].name + suffix (x) => x.title === aTblLinkColumns[i].name + suffix
)?.id; )?.id;
await sMap.addToMappingTbl( sMap.addToMappingTbl(
aTblLinkColumns[i].id, aTblLinkColumns[i].id,
ncId, ncId,
aTblLinkColumns[i].name + suffix, aTblLinkColumns[i].name + suffix,
@ -914,7 +913,7 @@ export default async (
// parent table ID // parent table ID
// let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
if (aTblColumns.length) { if (aTblColumns.length) {
@ -942,10 +941,10 @@ export default async (
continue; continue;
} }
const ncRelationColumnId = await sMap.getNcIdFromAtId( const ncRelationColumnId = sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.relationColumnId aTblColumns[i].typeOptions.relationColumnId
); );
const ncLookupColumnId = await sMap.getNcIdFromAtId( const ncLookupColumnId = sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.foreignTableRollupColumnId aTblColumns[i].typeOptions.foreignTableRollupColumnId
); );
@ -979,7 +978,7 @@ export default async (
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
(x) => x.title === aTblColumns[i].name (x) => x.title === aTblColumns[i].name
)?.id; )?.id;
await sMap.addToMappingTbl( sMap.addToMappingTbl(
aTblColumns[i].id, aTblColumns[i].id,
ncId, ncId,
aTblColumns[i].name, aTblColumns[i].name,
@ -1018,10 +1017,10 @@ export default async (
const srcTableId = nestedLookupTbl[0].srcTableId; const srcTableId = nestedLookupTbl[0].srcTableId;
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
const ncRelationColumnId = await sMap.getNcIdFromAtId( const ncRelationColumnId = sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.relationColumnId nestedLookupTbl[0].typeOptions.relationColumnId
); );
const ncLookupColumnId = await sMap.getNcIdFromAtId( const ncLookupColumnId = sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId
); );
@ -1059,7 +1058,7 @@ export default async (
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
(x) => x.title === nestedLookupTbl[0].name (x) => x.title === nestedLookupTbl[0].name
)?.id; )?.id;
await sMap.addToMappingTbl( sMap.addToMappingTbl(
nestedLookupTbl[0].id, nestedLookupTbl[0].id,
ncId, ncId,
nestedLookupTbl[0].name, nestedLookupTbl[0].name,
@ -1104,7 +1103,7 @@ export default async (
// parent table ID // parent table ID
// let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
if (aTblColumns.length) { if (aTblColumns.length) {
@ -1149,10 +1148,10 @@ export default async (
continue; continue;
} }
const ncRelationColumnId = await sMap.getNcIdFromAtId( const ncRelationColumnId = sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.relationColumnId aTblColumns[i].typeOptions.relationColumnId
); );
const ncRollupColumnId = await sMap.getNcIdFromAtId( const ncRollupColumnId = sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.foreignTableRollupColumnId aTblColumns[i].typeOptions.foreignTableRollupColumnId
); );
@ -1203,7 +1202,7 @@ export default async (
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
(x) => x.title === aTblColumns[i].name (x) => x.title === aTblColumns[i].name
)?.id; )?.id;
await sMap.addToMappingTbl( sMap.addToMappingTbl(
aTblColumns[i].id, aTblColumns[i].id,
ncId, ncId,
aTblColumns[i].name, aTblColumns[i].name,
@ -1222,10 +1221,10 @@ export default async (
const srcTableId = nestedLookupTbl[0].srcTableId; const srcTableId = nestedLookupTbl[0].srcTableId;
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
const ncRelationColumnId = await sMap.getNcIdFromAtId( const ncRelationColumnId = sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.relationColumnId nestedLookupTbl[0].typeOptions.relationColumnId
); );
const ncLookupColumnId = await sMap.getNcIdFromAtId( const ncLookupColumnId = sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId
); );
@ -1260,7 +1259,7 @@ export default async (
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
(x) => x.title === nestedLookupTbl[0].name (x) => x.title === nestedLookupTbl[0].name
)?.id; )?.id;
await sMap.addToMappingTbl( sMap.addToMappingTbl(
nestedLookupTbl[0].id, nestedLookupTbl[0].id,
ncId, ncId,
nestedLookupTbl[0].name, nestedLookupTbl[0].name,
@ -1281,7 +1280,7 @@ export default async (
); );
const pColId = aTblSchema[idx].primaryColumnId; const pColId = aTblSchema[idx].primaryColumnId;
const ncColId = await sMap.getNcIdFromAtId(pColId); const ncColId = sMap.getNcIdFromAtId(pColId);
// skip primary column configuration if we field not migrated // skip primary column configuration if we field not migrated
if (ncColId) { if (ncColId) {
@ -1291,7 +1290,7 @@ export default async (
recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
// update schema // update schema
const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const ncTblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
await updateNcTblSchemaById(ncTblId); await updateNcTblSchemaById(ncTblId);
} }
} }
@ -1581,7 +1580,7 @@ export default async (
async function nocoConfigureFormView(sDB, aTblSchema) { async function nocoConfigureFormView(sDB, aTblSchema) {
if (!sDB.options.syncViews) return; if (!sDB.options.syncViews) return;
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form'); const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form');
const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form;
@ -1653,7 +1652,7 @@ export default async (
async function nocoConfigureGridView(sDB, aTblSchema) { async function nocoConfigureGridView(sDB, aTblSchema) {
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid'); const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid');
let viewCnt = idx; let viewCnt = idx;
@ -1692,7 +1691,7 @@ export default async (
recordPerfStats(_perfStart, 'dbView.gridCreate'); recordPerfStats(_perfStart, 'dbView.gridCreate');
await updateNcTblSchemaById(tblId); await updateNcTblSchemaById(tblId);
await sMap.addToMappingTbl( sMap.addToMappingTbl(
gridViews[i].id, gridViews[i].id,
viewCreated.id, viewCreated.id,
viewName, viewName,
@ -1969,7 +1968,7 @@ export default async (
// one of not migrated column; // one of not migrated column;
if (!colSchema) { if (!colSchema) {
updateMigrationSkipLog( updateMigrationSkipLog(
await sMap.getNcNameFromAtId(viewId), sMap.getNcNameFromAtId(viewId),
colSchema.title, colSchema.title,
colSchema.uidt, colSchema.uidt,
`filter config skipped; column not migrated` `filter config skipped; column not migrated`
@ -1984,7 +1983,7 @@ export default async (
if (datatype === UITypes.Date || datatype === UITypes.DateTime) { if (datatype === UITypes.Date || datatype === UITypes.DateTime) {
// skip filters over data datatype // skip filters over data datatype
updateMigrationSkipLog( updateMigrationSkipLog(
await sMap.getNcNameFromAtId(viewId), sMap.getNcNameFromAtId(viewId),
colSchema.title, colSchema.title,
colSchema.uidt, colSchema.uidt,
`filter config skipped; filter over date datatype not supported` `filter config skipped; filter over date datatype not supported`
@ -2004,7 +2003,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: await sMap.getNcNameFromAtId(filter.value[i]), value: sMap.getNcNameFromAtId(filter.value[i]),
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -2015,7 +2014,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: await sMap.getNcNameFromAtId(filter.value), value: sMap.getNcNameFromAtId(filter.value),
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -2111,7 +2110,7 @@ export default async (
// rest of the columns from airtable- retain order & visibility property // rest of the columns from airtable- retain order & visibility property
for (let j = 0; j < c.length; j++) { for (let j = 0; j < c.length; j++) {
const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId); const ncColumnId = sMap.getNcIdFromAtId(c[j].columnId);
const ncViewColumnId = await nc_getViewColumnId( const ncViewColumnId = await nc_getViewColumnId(
viewId, viewId,
viewType, viewType,
@ -2257,7 +2256,7 @@ export default async (
sDB: syncDB, sDB: syncDB,
logDetailed, logDetailed,
}); });
rtc.data.records += await recordsMap[ncTbl.id].getCount(); rtc.data.records += recordsMap[ncTbl.id].getCount();
logDetailed(`Data inserted from ${ncTbl.title}`); logDetailed(`Data inserted from ${ncTbl.title}`);
} }

7
packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts

@ -33,14 +33,13 @@ async function readAllData({
async function page(records, fetchNextPage) { async function page(records, fetchNextPage) {
if (!data) { if (!data) {
data = new EntityMap(); data = new EntityMap();
await data.init();
} }
for await (const record of records) { for (const record of records) {
await data.addRow({ id: record.id, ...record.fields }); data.addRow({ id: record.id, ...record.fields });
} }
const tmpLength = await data.getCount(); const tmpLength = data.getCount();
logBasic( logBasic(
`:: Reading '${table.title}' data :: ${Math.max( `:: Reading '${table.title}' data :: ${Math.max(

4
packages/nocodb/src/lib/migrations/v2/nc_012_alter_column_data_types.ts

@ -2,7 +2,7 @@ import { Knex } from 'knex';
import { MetaTable } from '../../utils/globals'; import { MetaTable } from '../../utils/globals';
const up = async (knex: Knex) => { const up = async (knex: Knex) => {
if (knex.client.config.client !== 'sqlite3') { if (knex.client.config.client !== 'sqlite3' && knex.client.config.client !== 'better-sqlite3') {
await knex.schema.alterTable(MetaTable.COLUMNS, (table) => { await knex.schema.alterTable(MetaTable.COLUMNS, (table) => {
table.text('cdf').alter(); table.text('cdf').alter();
}); });
@ -19,7 +19,7 @@ const up = async (knex: Knex) => {
}; };
const down = async (knex) => { const down = async (knex) => {
if (knex.client.config.client !== 'sqlite3') { if (knex.client.config.client !== 'sqlite3' && knex.client.config.client !== 'better-sqlite3') {
await knex.schema.alterTable(MetaTable.COLUMNS, (table) => { await knex.schema.alterTable(MetaTable.COLUMNS, (table) => {
table.string('cdf').alter(); table.string('cdf').alter();
}); });

4
packages/nocodb/src/lib/migrations/v2/nc_014_alter_column_data_types.ts

@ -2,7 +2,7 @@ import { Knex } from 'knex';
import { MetaTable } from '../../utils/globals'; import { MetaTable } from '../../utils/globals';
const up = async (knex: Knex) => { const up = async (knex: Knex) => {
if (knex.client.config.client !== 'sqlite3') { if (knex.client.config.client !== 'sqlite3' && knex.client.config.client !== 'better-sqlite3') {
await knex.schema.alterTable(MetaTable.FORM_VIEW, (table) => { await knex.schema.alterTable(MetaTable.FORM_VIEW, (table) => {
table.text('success_msg').alter(); table.text('success_msg').alter();
}); });
@ -22,7 +22,7 @@ const up = async (knex: Knex) => {
}; };
const down = async (knex) => { const down = async (knex) => {
if (knex.client.config.client !== 'sqlite3') { if (knex.client.config.client !== 'sqlite3' && knex.client.config.client !== 'better-sqlite3') {
await knex.schema.alterTable(MetaTable.FORM_VIEW, (table) => { await knex.schema.alterTable(MetaTable.FORM_VIEW, (table) => {
table.string('success_msg').alter(); table.string('success_msg').alter();
}); });

4
packages/nocodb/src/lib/migrations/v2/nc_016_alter_hooklog_payload_types.ts

@ -9,7 +9,7 @@ const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.HOOK_LOGS, (table) => { await knex.schema.alterTable(MetaTable.HOOK_LOGS, (table) => {
table.text('payload'); table.text('payload');
}); });
} else if (knex.client.config.client !== 'sqlite3') { } else if (knex.client.config.client !== 'sqlite3' && knex.client.config.client !== 'better-sqlite3') {
await knex.schema.alterTable(MetaTable.HOOK_LOGS, (table) => { await knex.schema.alterTable(MetaTable.HOOK_LOGS, (table) => {
table.text('payload').alter(); table.text('payload').alter();
}); });
@ -17,7 +17,7 @@ const up = async (knex: Knex) => {
}; };
const down = async (knex) => { const down = async (knex) => {
if (knex.client.config.client !== 'sqlite3') { if (knex.client.config.client !== 'sqlite3' && knex.client.config.client !== 'better-sqlite3') {
await knex.schema.alterTable(MetaTable.HOOK_LOGS, (table) => { await knex.schema.alterTable(MetaTable.HOOK_LOGS, (table) => {
table.boolean('payload').alter(); table.boolean('payload').alter();
}); });

4
packages/nocodb/src/lib/models/Base.ts

@ -101,7 +101,7 @@ export default class Base implements BaseType {
if (this.is_meta) { if (this.is_meta) {
const metaConfig = Noco.getConfig()?.meta?.db; const metaConfig = Noco.getConfig()?.meta?.db;
const config = { ...metaConfig }; const config = { ...metaConfig };
if (config.client === 'sqlite3') { if (config.client === 'sqlite3' || config.client === 'better-sqlite3') {
config.connection = metaConfig; config.connection = metaConfig;
} }
@ -116,7 +116,7 @@ export default class Base implements BaseType {
); );
// todo: update sql-client args // todo: update sql-client args
if (config?.client === 'sqlite3') { if (config?.client === 'sqlite3' || config?.client === 'better-sqlite3') {
config.connection.filename = config.connection.filename =
config.connection.filename || config.connection?.connection.filename; config.connection.filename || config.connection?.connection.filename;
} }

26
packages/nocodb/src/lib/utils/NcConfigFactory.ts

@ -23,7 +23,7 @@ const driverClientMapping = {
mariadb: 'mysql2', mariadb: 'mysql2',
postgres: 'pg', postgres: 'pg',
postgresql: 'pg', postgresql: 'pg',
sqlite: 'sqlite3', sqlite: 'better-sqlite3',
mssql: 'mssql', mssql: 'mssql',
}; };
@ -132,7 +132,7 @@ export default class NcConfigFactory implements NcConfig {
if (process.env.NC_TRY) { if (process.env.NC_TRY) {
ncConfig.try = true; ncConfig.try = true;
ncConfig.meta.db = { ncConfig.meta.db = {
client: 'sqlite3', client: 'better-sqlite3',
connection: ':memory:', connection: ':memory:',
pool: { pool: {
min: 1, min: 1,
@ -189,11 +189,11 @@ export default class NcConfigFactory implements NcConfig {
let dbConfig: DbConfig; let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) { if (url.protocol.startsWith('sqlite3') || url.protocol.startsWith('better-sqlite3')) {
dbConfig = { dbConfig = {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: filename:
url.searchParams.get('d') || url.searchParams.get('database'), url.searchParams.get('d') || url.searchParams.get('database'),
@ -297,10 +297,10 @@ export default class NcConfigFactory implements NcConfig {
let dbConfig; let dbConfig;
if (url.protocol.startsWith('sqlite3')) { if (url.protocol.startsWith('sqlite3') || url.protocol.startsWith('better-sqlite3')) {
const db = url.searchParams.get('d') || url.searchParams.get('database'); const db = url.searchParams.get('d') || url.searchParams.get('database');
dbConfig = { dbConfig = {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: db, filename: db,
}, },
@ -437,7 +437,7 @@ export default class NcConfigFactory implements NcConfig {
if (process.env.NC_TRY) { if (process.env.NC_TRY) {
config.try = true; config.try = true;
config.meta.db = { config.meta.db = {
client: 'sqlite3', client: 'better-sqlite3',
connection: ':memory:', connection: ':memory:',
pool: { pool: {
min: 1, min: 1,
@ -490,9 +490,9 @@ export default class NcConfigFactory implements NcConfig {
const config = new NcConfigFactory(); const config = new NcConfigFactory();
let dbConfig = dbConnectionConfig; let dbConfig = dbConnectionConfig;
if (dbConfig.client === 'sqlite3') { if (dbConfig.client === 'sqlite3' || dbConfig.client === 'better-sqlite3') {
dbConfig = { dbConfig = {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
...dbConnectionConfig, ...dbConnectionConfig,
database: dbConnectionConfig.connection.filename, database: dbConnectionConfig.connection.filename,
@ -553,7 +553,7 @@ export default class NcConfigFactory implements NcConfig {
if (process.env.NC_TRY) { if (process.env.NC_TRY) {
config.try = true; config.try = true;
config.meta.db = { config.meta.db = {
client: 'sqlite3', client: 'better-sqlite3',
connection: ':memory:', connection: ':memory:',
pool: { pool: {
min: 1, min: 1,
@ -585,7 +585,7 @@ export default class NcConfigFactory implements NcConfig {
} }
public static async metaDbCreateIfNotExist(args: NcConfig) { public static async metaDbCreateIfNotExist(args: NcConfig) {
if (args.meta?.db?.client === 'sqlite3') { if (args.meta?.db?.client === 'sqlite3' || args.meta?.db?.client === 'better-sqlite3') {
const metaSqlClient = SqlClientFactory.create({ const metaSqlClient = SqlClientFactory.create({
...args.meta.db, ...args.meta.db,
connection: args.meta.db, connection: args.meta.db,
@ -624,7 +624,7 @@ export default class NcConfigFactory implements NcConfig {
public projectType; public projectType;
public meta = { public meta = {
db: { db: {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: 'noco.db', filename: 'noco.db',
}, },

2
packages/nocodb/src/lib/utils/common/NcConnectionMgr.ts

@ -95,7 +95,7 @@ export default class NcConnectionMgr {
} }
} }
const isSqlite = connectionConfig?.client === 'sqlite3'; const isSqlite = connectionConfig?.client === 'sqlite3' || connectionConfig?.client === 'better-sqlite3';
if (connectionConfig?.connection?.port) { if (connectionConfig?.connection?.port) {
connectionConfig.connection.port = +connectionConfig.connection.port; connectionConfig.connection.port = +connectionConfig.connection.port;

4
packages/nocodb/src/lib/v1-legacy/templates/NcTemplateParser.ts

@ -171,7 +171,7 @@ export default class NcTemplateParser {
onUpdate: 'NO ACTION', onUpdate: 'NO ACTION',
parentColumn: parentPrimaryColumn.cn, parentColumn: parentPrimaryColumn.cn,
parentTable: tableTemplate.tn, parentTable: tableTemplate.tn,
type: this.client === 'sqlite3' ? 'virtual' : 'real', type: this.client === 'sqlite3' || this.client === 'better-sqlite3' ? 'virtual' : 'real',
updateRelation: false, updateRelation: false,
}); });
} }
@ -213,7 +213,7 @@ export default class NcTemplateParser {
onUpdate: 'NO ACTION', onUpdate: 'NO ACTION',
parentColumn: parentPrimaryColumn.cn, parentColumn: parentPrimaryColumn.cn,
parentTable: parentTable.tn, parentTable: parentTable.tn,
type: this.client === 'sqlite3' ? 'virtual' : 'real', type: this.client === 'sqlite3' || this.client === 'better-sqlite3' ? 'virtual' : 'real',
updateRelation: false, updateRelation: false,
}); });
} }

4
packages/nocodb/tests/unit/TestDbMngr.ts

@ -142,7 +142,7 @@ export default class TestDbMngr {
static async switchToSqlite() { static async switchToSqlite() {
// process.env[`DATABASE_URL`] = `sqlite3:///?database=${__dirname}/${TestDbMngr.dbName}.sqlite`; // process.env[`DATABASE_URL`] = `sqlite3:///?database=${__dirname}/${TestDbMngr.dbName}.sqlite`;
TestDbMngr.dbConfig = { TestDbMngr.dbConfig = {
client: 'sqlite3', client: 'better-sqlite3',
connection: { connection: {
filename: `${__dirname}/${TestDbMngr.dbName}.db`, filename: `${__dirname}/${TestDbMngr.dbName}.db`,
database: TestDbMngr.dbName, database: TestDbMngr.dbName,
@ -182,7 +182,7 @@ export default class TestDbMngr {
} }
static isSqlite() { static isSqlite() {
return TestDbMngr.dbConfig.client === 'sqlite3'; return TestDbMngr.dbConfig.client === 'sqlite3' || TestDbMngr.dbConfig.client === 'better-sqlite3';
} }
private static async useDatabase(knexClient, dbName) { private static async useDatabase(knexClient, dbName) {

2
packages/nocodb/tests/unit/init/db.ts

@ -2,7 +2,7 @@ import { DbConfig } from "../../../src/interface/config";
const isSqlite = (context) =>{ const isSqlite = (context) =>{
return (context.dbConfig as DbConfig).client === 'sqlite' || (context.dbConfig as DbConfig).client === 'sqlite3'; return (context.dbConfig as DbConfig).client === 'sqlite' || (context.dbConfig as DbConfig).client === 'sqlite3' || (context.dbConfig as DbConfig).client === 'better-sqlite3';
} }
const isMysql = (context) => const isMysql = (context) =>

Loading…
Cancel
Save