Browse Source

chore(nocodb): lint

pull/4716/head
Wing-Kam Wong 2 years ago
parent
commit
4113a2c3d4
  1. 168
      packages/nocodb/src/lib/db/sql-client/lib/snowflake/SnowflakeClient.ts
  2. 6
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/CustomKnex.ts
  3. 33
      packages/nocodb/src/lib/meta/api/baseApis.ts
  4. 2
      packages/nocodb/src/lib/meta/api/projectApis.ts
  5. 6
      packages/nocodb/src/lib/meta/api/sync/helpers/fetchAT.ts
  6. 15
      packages/nocodb/src/lib/meta/api/sync/helpers/job.ts
  7. 8
      packages/nocodb/src/lib/meta/api/sync/syncSourceApis.ts
  8. 5
      packages/nocodb/src/lib/meta/helpers/extractProps.ts
  9. 5
      packages/nocodb/src/lib/meta/helpers/syncMigration.ts
  10. 2
      packages/nocodb/src/lib/migrations/v2/nc_023_multiple_source.ts
  11. 4
      packages/nocodb/src/lib/models/SyncSource.ts

168
packages/nocodb/src/lib/db/sql-client/lib/snowflake/SnowflakeClient.ts

@ -9,14 +9,14 @@ import lodash from 'lodash';
const log = new Debug('SnowflakeClient');
const rowsToLower = (arr) => {
for(const a of arr) {
for(const [k, v] of Object.entries(a)) {
for (const a of arr) {
for (const [k, v] of Object.entries(a)) {
delete a[k];
a[k.toLowerCase()] = v;
}
}
return arr;
}
};
class SnowflakeClient extends KnexClient {
private queries: any;
@ -228,7 +228,7 @@ class SnowflakeClient extends KnexClient {
* @returns {Number} code
* @returns {String} message
*/
async testConnection(args: any = {}) {
async testConnection(args: any = {}) {
const func = this.testConnection.name;
const result = new Result();
log.api(`${func}:args:`, args);
@ -253,7 +253,7 @@ class SnowflakeClient extends KnexClient {
getKnexDataTypes() {
const result = new Result();
result.data.list = [
result.data.list = [
'NUMBER',
'DECIMAL',
'NUMERIC',
@ -309,7 +309,9 @@ class SnowflakeClient extends KnexClient {
try {
result.data.object = {};
const data = await this.sqlClient.raw('SELECT CURRENT_VERSION() as "server_version"');
const data = await this.sqlClient.raw(
'SELECT CURRENT_VERSION() as "server_version"'
);
log.debug(data.rows[0]);
result.data.object.version = data.rows[0].server_version;
const versions = data.rows[0].server_version.split('.');
@ -363,8 +365,10 @@ class SnowflakeClient extends KnexClient {
await this.sqlClient.raw(`CREATE DATABASE ??`, [args.database]);
}
await this.sqlClient.raw(`CREATE SCHEMA IF NOT EXISTS ??.??`,[args.database, this.schema]);
await this.sqlClient.raw(`CREATE SCHEMA IF NOT EXISTS ??.??`, [
args.database,
this.schema,
]);
} catch (e) {
log.ppe(e, _func);
throw e;
@ -549,8 +553,9 @@ class SnowflakeClient extends KnexClient {
log.api(`${_func}:args:`, args);
try {
const { rows } = await this
.raw(`SELECT SCHEMA_NAME as "schema_name" FROM "${this.database}".information_schema.SCHEMATA order by schema_name;`);
const { rows } = await this.raw(
`SELECT SCHEMA_NAME as "schema_name" FROM "${this.database}".information_schema.SCHEMATA order by schema_name;`
);
result.data.list = rows;
} catch (e) {
@ -595,22 +600,30 @@ class SnowflakeClient extends KnexClient {
log.api(`${_func}:args:`, args);
try {
args.databaseName = this.database;
await this.sqlClient.raw(`SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`);
await this.sqlClient.raw(`SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`);
await this.sqlClient.raw(
`SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`
);
await this.sqlClient.raw(
`SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`
);
const lastQueries = await this.sqlClient.raw(`
select * from table("${this.database}".information_schema.query_history())
WHERE query_text like 'SHOW%'
ORDER BY start_time DESC
LIMIT 30;`
);
LIMIT 30;`);
let pk_query_id, uq_query_id;
for (const r of lastQueries.rows) {
if (r.QUERY_TEXT === `SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`) {
if (
r.QUERY_TEXT ===
`SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`
) {
pk_query_id = r.QUERY_ID;
} else if (r.QUERY_TEXT === `SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`) {
} else if (
r.QUERY_TEXT ===
`SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`
) {
uq_query_id = r.QUERY_ID;
}
if (pk_query_id && uq_query_id) {
@ -766,21 +779,30 @@ class SnowflakeClient extends KnexClient {
const result = new Result();
log.api(`${_func}:args:`, args);
try {
await this.sqlClient.raw(`SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`);
await this.sqlClient.raw(`SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`);
await this.sqlClient.raw(
`SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`
);
await this.sqlClient.raw(
`SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`
);
const lastQueries = await this.sqlClient.raw(`
select * from table("${this.database}".INFORMATION_SCHEMA.query_history())
WHERE query_text like 'SHOW%'
ORDER BY start_time DESC
LIMIT 30;`
);
LIMIT 30;`);
let pk_query_id, uq_query_id;
for (const r of lastQueries.rows) {
if (r.QUERY_TEXT === `SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`) {
if (
r.QUERY_TEXT ===
`SHOW PRIMARY KEYS IN SCHEMA "${this.database}"."${this.schema}";`
) {
pk_query_id = r.QUERY_ID;
} else if (r.QUERY_TEXT === `SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`) {
} else if (
r.QUERY_TEXT ===
`SHOW UNIQUE KEYS IN SCHEMA "${this.database}"."${this.schema}";`
) {
uq_query_id = r.QUERY_ID;
}
if (pk_query_id && uq_query_id) {
@ -797,7 +819,8 @@ class SnowflakeClient extends KnexClient {
LEFT JOIN (select * from table(result_scan('${pk_query_id}')) UNION select * from table(result_scan('${uq_query_id}'))) pk
ON "PK"."constraint_name" = tc.constraint_name
WHERE tc.table_catalog = ? and tc.table_schema = ? and tc.table_name = ?;`,
[this.database, this.schema, args.tn]);
[this.database, this.schema, args.tn]
);
result.data.list = rows;
} catch (e) {
@ -826,23 +849,27 @@ class SnowflakeClient extends KnexClient {
* @property {String} - relations[].dr -
* @property {String} - relations[].mo -
*/
async relationList(args: any = {}) {
async relationList(args: any = {}) {
const _func = this.relationList.name;
const result = new Result();
log.api(`${_func}:args:`, args);
try {
await this.sqlClient.raw(`SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`);
await this.sqlClient.raw(
`SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`
);
const lastQueries = await this.sqlClient.raw(`
select * from table("${this.database}".information_schema.query_history())
WHERE query_text like 'SHOW%'
ORDER BY start_time DESC
LIMIT 30;`
);
LIMIT 30;`);
let ik_query_id;
for (const r of lastQueries.rows) {
if (r.QUERY_TEXT === `SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`) {
if (
r.QUERY_TEXT ===
`SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`
) {
ik_query_id = r.QUERY_ID;
break;
}
@ -897,18 +924,22 @@ class SnowflakeClient extends KnexClient {
const result = new Result();
log.api(`${_func}:args:`, args);
try {
await this.sqlClient.raw(`SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`);
await this.sqlClient.raw(
`SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`
);
const lastQueries = await this.sqlClient.raw(`
select * from table("${this.database}".information_schema.query_history())
WHERE query_text like 'SHOW%'
ORDER BY start_time DESC
LIMIT 30;`
);
LIMIT 30;`);
let ik_query_id;
for (const r of lastQueries.rows) {
if (r.QUERY_TEXT === `SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`) {
if (
r.QUERY_TEXT ===
`SHOW IMPORTED KEYS IN SCHEMA "${this.database}"."${this.schema}";`
) {
ik_query_id = r.QUERY_ID;
break;
}
@ -1060,7 +1091,7 @@ class SnowflakeClient extends KnexClient {
try {
const { rows } = await this.sqlClient.raw(
`select * from "${this.database}".INFORMATION_SCHEMA.views WHERE table_schema = ?;`,
[this.schema]
[this.schema]
);
for (let i = 0; i < rows.length; ++i) {
@ -1434,7 +1465,9 @@ class SnowflakeClient extends KnexClient {
throw new Error('Function not supported for Snowflake yet');
const upQuery =
this.querySeparator() +
`CREATE TRIGGER \`${args.procedure_name}\` \n${args.timing} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`;
`CREATE TRIGGER \`${args.procedure_name}\` \n${args.timing} ${
args.event
}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`;
await this.sqlClient.raw(upQuery);
const downQuery =
this.querySeparator() +
@ -1472,7 +1505,9 @@ class SnowflakeClient extends KnexClient {
this.querySeparator() + `DROP TRIGGER ${args.procedure_name}`;
const upQuery =
this.querySeparator() +
`CREATE TRIGGER \`${args.procedure_name}\` \n${args.timing} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`;
`CREATE TRIGGER \`${args.procedure_name}\` \n${args.timing} ${
args.event
}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`;
await this.sqlClient.raw(query);
await this.sqlClient.raw(upQuery);
@ -1507,7 +1542,9 @@ class SnowflakeClient extends KnexClient {
throw new Error('Function not supported for Snowflake yet');
const upQuery =
this.querySeparator() +
`CREATE TRIGGER ${args.trigger_name} \n${args.timing} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`;
`CREATE TRIGGER ${args.trigger_name} \n${args.timing} ${
args.event
}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`;
await this.sqlClient.raw(upQuery);
result.data.object = {
upStatement: [{ sql: upQuery }],
@ -1544,7 +1581,9 @@ class SnowflakeClient extends KnexClient {
`DROP TRIGGER ${args.trigger_name} ON ${args.tn}`
);
await this.sqlClient.raw(
`CREATE TRIGGER ${args.trigger_name} \n${args.timing} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`
`CREATE TRIGGER ${args.trigger_name} \n${args.timing} ${
args.event
}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`
);
result.data.object = {
@ -1554,10 +1593,14 @@ class SnowflakeClient extends KnexClient {
args.tn
};${this.querySeparator()}CREATE TRIGGER ${args.trigger_name} \n${
args.timing
} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.statement}`,
} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${
args.statement
}`,
downStatement:
this.querySeparator() +
`CREATE TRIGGER ${args.trigger_name} \n${args.timing} ${args.event}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.oldStatement}`,
`CREATE TRIGGER ${args.trigger_name} \n${args.timing} ${
args.event
}\nON ${this.getTnPath(args.tn)} FOR EACH ROW\n${args.oldStatement}`,
};
} catch (e) {
log.ppe(e, func);
@ -1712,7 +1755,9 @@ class SnowflakeClient extends KnexClient {
const downStatement =
this.querySeparator() +
this.sqlClient.raw(`DROP TABLE ??`, [this.getTnPath(args.table)]).toString();
this.sqlClient
.raw(`DROP TABLE ??`, [this.getTnPath(args.table)])
.toString();
this.emit(`Success : ${upQuery}`);
@ -1857,7 +1902,7 @@ class SnowflakeClient extends KnexClient {
//downQuery = `ALTER TABLE "${args.columns[0].tn}" ${downQuery};`;
}
for(const q of upQuery.split(';')) {
for (const q of upQuery.split(';')) {
if (q && q.trim() !== '') {
await this.sqlClient.raw(q);
}
@ -1902,7 +1947,9 @@ class SnowflakeClient extends KnexClient {
/** ************** create up & down statements *************** */
const upStatement =
this.querySeparator() +
this.sqlClient.raw(`DROP TABLE ??`, [this.getTnPath(args.tn)]).toString();
this.sqlClient
.raw(`DROP TABLE ??`, [this.getTnPath(args.tn)])
.toString();
let downQuery = this.createTable(args.tn, args);
/**
@ -2007,7 +2054,7 @@ class SnowflakeClient extends KnexClient {
* @param {String} - args.childTable
* @returns {Promise<{upStatement, downStatement}>}
*/
async relationCreate(args) {
async relationCreate(args) {
const _func = this.relationCreate.name;
const result = new Result();
log.api(`${_func}:args:`, args);
@ -2371,7 +2418,10 @@ class SnowflakeClient extends KnexClient {
: '';
if (numOfPksInNew.length) {
if (createTable) {
query += this.genQuery(`, constraint ?? PRIMARY KEY (??)`, [`${t}_pkey`, numOfPksInNew]);
query += this.genQuery(`, constraint ?? PRIMARY KEY (??)`, [
`${t}_pkey`,
numOfPksInNew,
]);
} else {
query += this.genQuery(
`alter TABLE ?? add constraint ?? PRIMARY KEY(??);`,
@ -2416,7 +2466,9 @@ class SnowflakeClient extends KnexClient {
query += this.alterTablePK(table, args.columns, [], query, true);
query = this.genQuery(`CREATE TABLE ?? (${query});`, [this.getTnPath(args.tn)]);
query = this.genQuery(`CREATE TABLE ?? (${query});`, [
this.getTnPath(args.tn),
]);
return query;
}
@ -2430,7 +2482,11 @@ class SnowflakeClient extends KnexClient {
if (change === 0) {
query = existingQuery ? ',' : '';
if (n.ai) {
query += this.genQuery(` ?? NUMBER(38,0) NOT NULL autoincrement UNIQUE`, [n.cn], shouldSanitize);
query += this.genQuery(
` ?? NUMBER(38,0) NOT NULL autoincrement UNIQUE`,
[n.cn],
shouldSanitize
);
} else {
query += this.genQuery(` ?? ${n.dt}`, [n.cn], shouldSanitize);
query += n.dtxp && n.dt !== 'text' ? `(${n.dtxp})` : '';
@ -2442,7 +2498,11 @@ class SnowflakeClient extends KnexClient {
query += n.dtxp && n.dt !== 'text' ? `(${n.dtxp})` : '';
query += n.rqd ? ' NOT NULL' : ' NULL';
query += defaultValue ? ` DEFAULT ${defaultValue}` : '';
query = this.genQuery(`ALTER TABLE ?? ${query};`, [this.getTnPath(t)], shouldSanitize);
query = this.genQuery(
`ALTER TABLE ?? ${query};`,
[this.getTnPath(t)],
shouldSanitize
);
} else {
if (n.cn !== o.cn) {
query += this.genQuery(
@ -2483,17 +2543,11 @@ class SnowflakeClient extends KnexClient {
}
get schema() {
return (
(this.connectionConfig &&
this.connectionConfig.connection.schema)
);
return this.connectionConfig && this.connectionConfig.connection.schema;
}
get database() {
return (
(this.connectionConfig &&
this.connectionConfig.connection.database)
);
return this.connectionConfig && this.connectionConfig.connection.database;
}
getTnPath(t) {

6
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/CustomKnex.ts

@ -1022,9 +1022,9 @@ function CustomKnex(arg: string | Knex.Config<any> | any): CustomKnex {
value: () => {
return typeof arg === 'string'
? arg.match(/^(\w+):/) ?? [1]
: (typeof arg.client === 'string')
? arg.client
: arg.client?.prototype?.dialect || arg.client?.prototype?.driverName;
: typeof arg.client === 'string'
? arg.client
: arg.client?.prototype?.dialect || arg.client?.prototype?.driverName;
},
},
searchPath: {

33
packages/nocodb/src/lib/meta/api/baseApis.ts

@ -35,19 +35,17 @@ export async function baseUpdate(
) {
const baseBody = req.body;
const project = await Project.getWithInfo(req.params.projectId);
const base = await Base.updateBase(req.params.baseId,
{
...baseBody,
type: baseBody.config?.client,
projectId: project.id,
id: req.params.baseId,
}
);
const base = await Base.updateBase(req.params.baseId, {
...baseBody,
type: baseBody.config?.client,
projectId: project.id,
id: req.params.baseId,
});
delete base.config;
Tele.emit('evt', {
evt_type: 'base:updated'
evt_type: 'base:updated',
});
res.json(base);
@ -63,12 +61,11 @@ export async function baseList(
res // todo: pagination
.json({
bases: new PagedResponseImpl(bases, {
count: bases.length,
limit: bases.length,
})
}
);
bases: new PagedResponseImpl(bases, {
count: bases.length,
limit: bases.length,
}),
});
} catch (e) {
console.log(e);
next(e);
@ -98,13 +95,13 @@ async function baseCreate(req: Request<any, any>, res) {
await syncBaseMigration(project, base);
const info = await populateMeta(base, project);
Tele.emit('evt_api_created', info);
delete base.config;
Tele.emit('evt', {
evt_type: 'base:created'
evt_type: 'base:created',
});
res.json(base);
@ -142,14 +139,12 @@ async function populateMeta(base: Base, project: Project): Promise<any> {
return t;
});
/* filter based on prefix */
if (base.is_meta && project?.prefix) {
tables = tables.filter((t) => {
return t?.tn?.startsWith(project?.prefix);
});
}
info.tablesCount = tables.length;

2
packages/nocodb/src/lib/meta/api/projectApis.ts

@ -465,7 +465,7 @@ export async function projectInfoGet(_req, res) {
export async function projectCost(req, res) {
let cost = 0;
const project = await Project.getWithInfo(req.params.projectId);
for (const base of project.bases) {
const sqlClient = NcConnectionMgrv2.getSqlClient(base);
const userCount = await ProjectUser.getUsersCount(req.query);

6
packages/nocodb/src/lib/meta/api/sync/helpers/fetchAT.ts

@ -121,8 +121,7 @@ async function read() {
};
} else {
throw {
message:
'Error Initializing :: please try again !!',
message: 'Error Initializing :: please try again !!',
};
}
}
@ -177,8 +176,7 @@ async function readView(viewId) {
return { view: resreq.data };
} else {
throw {
message:
'Error Initializing :: please try again !!',
message: 'Error Initializing :: please try again !!',
};
}
}

15
packages/nocodb/src/lib/meta/api/sync/helpers/job.ts

@ -190,7 +190,7 @@ export default async (
const duration = Date.now() - start;
rtc.fetchAt.count++;
rtc.fetchAt.time += duration;
if (!ft.baseId) {
throw {
message:
@ -221,7 +221,8 @@ export default async (
}
function getRootDbType() {
return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId)?.type;
return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId)
?.type;
}
// base mapping table
@ -319,7 +320,10 @@ export default async (
// @ts-ignore
async function nc_DumpTableSchema() {
console.log('[');
const ncTblList = await api.base.tableList(ncCreatedProjectSchema.id, syncDB.baseId);
const ncTblList = await api.base.tableList(
ncCreatedProjectSchema.id,
syncDB.baseId
);
for (let i = 0; i < ncTblList.list.length; i++) {
const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
console.log(JSON.stringify(ncTbl, null, 2));
@ -2237,7 +2241,10 @@ export default async (
try {
// await nc_DumpTableSchema();
const _perfStart = recordPerfStart();
const ncTblList = await api.base.tableList(ncCreatedProjectSchema.id, syncDB.baseId);
const ncTblList = await api.base.tableList(
ncCreatedProjectSchema.id,
syncDB.baseId
);
recordPerfStats(_perfStart, 'base.tableList');
logBasic('Reading Records...');

8
packages/nocodb/src/lib/meta/api/sync/syncSourceApis.ts

@ -8,13 +8,17 @@ import Project from '../../../models/Project';
export async function syncSourceList(req: Request, res: Response) {
// todo: pagination
res.json(new PagedResponseImpl(await SyncSource.list(req.params.projectId, req.params.baseId)));
res.json(
new PagedResponseImpl(
await SyncSource.list(req.params.projectId, req.params.baseId)
)
);
}
export async function syncCreate(req: Request, res: Response) {
Tele.emit('evt', { evt_type: 'webhooks:created' });
const project = await Project.getWithInfo(req.params.projectId);
const sync = await SyncSource.insert({
...req.body,
fk_user_id: (req as any).user.id,

5
packages/nocodb/src/lib/meta/helpers/extractProps.ts

@ -1,6 +1,9 @@
import DOMPurify from 'isomorphic-dompurify';
export function extractProps<T extends object>(body: T, props: string[]): Partial<T> {
export function extractProps<T extends object>(
body: T,
props: string[]
): Partial<T> {
// todo: throw error if no props found
return props.reduce((o, key) => {
if (key in body) o[key] = body[key];

5
packages/nocodb/src/lib/meta/helpers/syncMigration.ts

@ -21,7 +21,10 @@ export default async function syncMigration(project: Project): Promise<void> {
}
}
export async function syncBaseMigration(project: Project, base: Base): Promise<void> {
export async function syncBaseMigration(
project: Project,
base: Base
): Promise<void> {
try {
/* create sql-migrator */
const migrator = new Migrator(project);

2
packages/nocodb/src/lib/migrations/v2/nc_023_multiple_source.ts

@ -21,7 +21,7 @@ const down = async (knex) => {
await knex.schema.alterTable(MetaTable.SYNC_SOURCE, (table) => {
table.dropColumn('base_id');
});
await knex.schema.alterTable(MetaTable.BASES, (table) => {
table.dropColumn('enabled');
table.dropColumn('order');

4
packages/nocodb/src/lib/models/SyncSource.ts

@ -39,7 +39,9 @@ export default class SyncSource {
}
static async list(projectId: string, baseId?: string, ncMeta = Noco.ncMeta) {
const condition = baseId ? { project_id: projectId, base_id: baseId } : { project_id: projectId };
const condition = baseId
? { project_id: projectId, base_id: baseId }
: { project_id: projectId };
const syncSources = await ncMeta.metaList(
null,
null,

Loading…
Cancel
Save