Browse Source

feat: use id instead of alias on queries

pull/7060/head
mertmit 9 months ago
parent
commit
d9b03d3526
  1. 2
      packages/nocodb-sdk/src/lib/formulaHelpers.ts
  2. 174
      packages/nocodb/src/db/BaseModelSqlv2.ts

2
packages/nocodb-sdk/src/lib/formulaHelpers.ts

@ -93,7 +93,7 @@ export function substituteColumnIdWithAliasInFormula(
c.column_name === colNameOrId || c.column_name === colNameOrId ||
c.title === colNameOrId c.title === colNameOrId
); );
pt.name = column?.title || ptRaw?.name || pt?.name; pt.name = column?.id || ptRaw?.name || pt?.name;
} else if (pt.type === 'BinaryExpression') { } else if (pt.type === 'BinaryExpression') {
substituteId(pt.left, ptRaw?.left); substituteId(pt.left, ptRaw?.left);
substituteId(pt.right, ptRaw?.right); substituteId(pt.right, ptRaw?.right);

174
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -167,7 +167,9 @@ class BaseModelSqlv2 {
let data; let data;
try { try {
data = await this.execAndParse(qb, null, { first: true }); data = await this.execAndParse(qb, null, {
first: true,
});
} catch (e) { } catch (e) {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
throw e; throw e;
@ -587,9 +589,9 @@ class BaseModelSqlv2 {
knex: this.dbDriver, knex: this.dbDriver,
columnOptions: (await column.getColOptions()) as RollupColumn, columnOptions: (await column.getColOptions()) as RollupColumn,
}) })
).builder.as(sanitize(column.title)), ).builder.as(sanitize(column.id)),
); );
groupBySelectors.push(sanitize(column.title)); groupBySelectors.push(sanitize(column.id));
break; break;
case UITypes.Formula: case UITypes.Formula:
{ {
@ -601,18 +603,18 @@ class BaseModelSqlv2 {
selectQb = this.dbDriver.raw(`?? as ??`, [ selectQb = this.dbDriver.raw(`?? as ??`, [
_selectQb.builder, _selectQb.builder,
sanitize(column.title), sanitize(column.id),
]); ]);
} catch (e) { } catch (e) {
console.log(e); console.log(e);
// return dummy select // return dummy select
selectQb = this.dbDriver.raw(`'ERR' as ??`, [ selectQb = this.dbDriver.raw(`'ERR' as ??`, [
sanitize(column.title), sanitize(column.id),
]); ]);
} }
selectors.push(selectQb); selectors.push(selectQb);
groupBySelectors.push(column.title); groupBySelectors.push(column.id);
} }
break; break;
case UITypes.Lookup: case UITypes.Lookup:
@ -628,18 +630,18 @@ class BaseModelSqlv2 {
const selectQb = this.dbDriver.raw(`?? as ??`, [ const selectQb = this.dbDriver.raw(`?? as ??`, [
this.dbDriver.raw(_selectQb.builder).wrap('(', ')'), this.dbDriver.raw(_selectQb.builder).wrap('(', ')'),
sanitize(column.title), sanitize(column.id),
]); ]);
selectors.push(selectQb); selectors.push(selectQb);
groupBySelectors.push(sanitize(column.title)); groupBySelectors.push(sanitize(column.id));
} }
break; break;
default: default:
selectors.push( selectors.push(
this.dbDriver.raw('?? as ??', [column.column_name, column.title]), this.dbDriver.raw('?? as ??', [column.column_name, column.id]),
); );
groupBySelectors.push(sanitize(column.title)); groupBySelectors.push(sanitize(column.id));
break; break;
} }
}), }),
@ -704,7 +706,7 @@ class BaseModelSqlv2 {
} }
qb.orderBy( qb.orderBy(
groupByColumns[sort.fk_column_id].title, groupByColumns[sort.fk_column_id].id,
sort.direction, sort.direction,
sort.direction === 'desc' ? 'LAST' : 'FIRST', sort.direction === 'desc' ? 'LAST' : 'FIRST',
); );
@ -772,9 +774,9 @@ class BaseModelSqlv2 {
columnOptions: columnOptions:
(await column.getColOptions()) as RollupColumn, (await column.getColOptions()) as RollupColumn,
}) })
).builder.as(sanitize(column.title)), ).builder.as(sanitize(column.id)),
); );
groupBySelectors.push(sanitize(column.title)); groupBySelectors.push(sanitize(column.id));
break; break;
case UITypes.Formula: { case UITypes.Formula: {
let selectQb; let selectQb;
@ -785,18 +787,18 @@ class BaseModelSqlv2 {
selectQb = this.dbDriver.raw(`?? as ??`, [ selectQb = this.dbDriver.raw(`?? as ??`, [
_selectQb.builder, _selectQb.builder,
sanitize(column.title), sanitize(column.id),
]); ]);
} catch (e) { } catch (e) {
console.log(e); console.log(e);
// return dummy select // return dummy select
selectQb = this.dbDriver.raw(`'ERR' as ??`, [ selectQb = this.dbDriver.raw(`'ERR' as ??`, [
sanitize(column.title), sanitize(column.id),
]); ]);
} }
selectors.push(selectQb); selectors.push(selectQb);
groupBySelectors.push(column.title); groupBySelectors.push(column.id);
break; break;
} }
case UITypes.Lookup: case UITypes.Lookup:
@ -812,21 +814,18 @@ class BaseModelSqlv2 {
const selectQb = this.dbDriver.raw(`?? as ??`, [ const selectQb = this.dbDriver.raw(`?? as ??`, [
this.dbDriver.raw(_selectQb.builder).wrap('(', ')'), this.dbDriver.raw(_selectQb.builder).wrap('(', ')'),
sanitize(column.title), sanitize(column.id),
]); ]);
selectors.push(selectQb); selectors.push(selectQb);
groupBySelectors.push(sanitize(column.title)); groupBySelectors.push(sanitize(column.id));
} }
break; break;
default: default:
selectors.push( selectors.push(
this.dbDriver.raw('?? as ??', [ this.dbDriver.raw('?? as ??', [column.column_name, column.id]),
column.column_name,
column.title,
]),
); );
groupBySelectors.push(sanitize(column.title)); groupBySelectors.push(sanitize(column.id));
break; break;
} }
}), }),
@ -2099,11 +2098,10 @@ class BaseModelSqlv2 {
// the value 2023-01-01 10:00:00 (UTC) would display as 2023-01-01 18:00:00 (UTC+8) // the value 2023-01-01 10:00:00 (UTC) would display as 2023-01-01 18:00:00 (UTC+8)
// our existing logic is based on UTC, during the query, we need to take the UTC value // our existing logic is based on UTC, during the query, we need to take the UTC value
// hence, we use CONVERT_TZ to convert back to UTC value // hence, we use CONVERT_TZ to convert back to UTC value
res[sanitize(column.title || column.column_name)] = res[sanitize(column.id || column.column_name)] = this.dbDriver.raw(
this.dbDriver.raw( `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`,
`CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`, [`${sanitize(alias || this.tnPath)}.${column.column_name}`],
[`${sanitize(alias || this.tnPath)}.${column.column_name}`], );
);
break; break;
} else if (this.isPg) { } else if (this.isPg) {
// if there is no timezone info, // if there is no timezone info,
@ -2113,7 +2111,7 @@ class BaseModelSqlv2 {
column.dt !== 'timestamp with time zone' && column.dt !== 'timestamp with time zone' &&
column.dt !== 'timestamptz' column.dt !== 'timestamptz'
) { ) {
res[sanitize(column.title || column.column_name)] = this.dbDriver res[sanitize(column.id || column.column_name)] = this.dbDriver
.raw( .raw(
`?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`, `?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`,
[`${sanitize(alias || this.tnPath)}.${column.column_name}`], [`${sanitize(alias || this.tnPath)}.${column.column_name}`],
@ -2126,7 +2124,7 @@ class BaseModelSqlv2 {
// convert to database timezone, // convert to database timezone,
// then convert to UTC // then convert to UTC
if (column.dt !== 'datetimeoffset') { if (column.dt !== 'datetimeoffset') {
res[sanitize(column.title || column.column_name)] = res[sanitize(column.id || column.column_name)] =
this.dbDriver.raw( this.dbDriver.raw(
`CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`, `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
[`${sanitize(alias || this.tnPath)}.${column.column_name}`], [`${sanitize(alias || this.tnPath)}.${column.column_name}`],
@ -2134,7 +2132,7 @@ class BaseModelSqlv2 {
break; break;
} }
} }
res[sanitize(column.title || column.column_name)] = sanitize( res[sanitize(column.id || column.column_name)] = sanitize(
`${alias || this.tnPath}.${column.column_name}`, `${alias || this.tnPath}.${column.column_name}`,
); );
break; break;
@ -2197,7 +2195,7 @@ class BaseModelSqlv2 {
aliasToColumnBuilder, aliasToColumnBuilder,
); );
qb.select({ qb.select({
[column.title]: selectQb.builder, [column.id]: selectQb.builder,
}); });
} catch { } catch {
continue; continue;
@ -2205,7 +2203,7 @@ class BaseModelSqlv2 {
break; break;
default: { default: {
qb.select({ qb.select({
[column.title]: barcodeValueColumn.column_name, [column.id]: barcodeValueColumn.column_name,
}); });
break; break;
} }
@ -2225,14 +2223,14 @@ class BaseModelSqlv2 {
qb.select( qb.select(
this.dbDriver.raw(`?? as ??`, [ this.dbDriver.raw(`?? as ??`, [
selectQb.builder, selectQb.builder,
sanitize(column.title), sanitize(column.id),
]), ]),
); );
} catch (e) { } catch (e) {
console.log(e); console.log(e);
// return dummy select // return dummy select
qb.select( qb.select(
this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)]), this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.id)]),
); );
} }
} }
@ -2249,13 +2247,13 @@ class BaseModelSqlv2 {
alias, alias,
columnOptions: (await column.getColOptions()) as RollupColumn, columnOptions: (await column.getColOptions()) as RollupColumn,
}) })
).builder.as(sanitize(column.title)), ).builder.as(sanitize(column.id)),
); );
break; break;
default: default:
if (this.isPg) { if (this.isPg) {
if (column.dt === 'bytea') { if (column.dt === 'bytea') {
res[sanitize(column.title || column.column_name)] = res[sanitize(column.id || column.column_name)] =
this.dbDriver.raw( this.dbDriver.raw(
`encode(??.??, '${ `encode(??.??, '${
column.meta?.format === 'hex' ? 'hex' : 'escape' column.meta?.format === 'hex' ? 'hex' : 'escape'
@ -2266,7 +2264,7 @@ class BaseModelSqlv2 {
} }
} }
res[sanitize(column.title || column.column_name)] = sanitize( res[sanitize(column.id || column.column_name)] = sanitize(
`${alias || this.tnPath}.${column.column_name}`, `${alias || this.tnPath}.${column.column_name}`,
); );
break; break;
@ -2315,9 +2313,9 @@ class BaseModelSqlv2 {
const query = this.dbDriver(this.tnPath).insert(insertObj); const query = this.dbDriver(this.tnPath).insert(insertObj);
if ((this.isPg || this.isMssql) && this.model.primaryKey) { if ((this.isPg || this.isMssql) && this.model.primaryKey) {
query.returning( query.returning(
`${this.model.primaryKey.column_name} as ${this.model.primaryKey.title}`, `${this.model.primaryKey.column_name} as ${this.model.primaryKey.id}`,
); );
response = await this.execAndParse(query); response = await this.execAndParse(query, null, { raw: true });
} }
const ai = this.model.columns.find((c) => c.ai); const ai = this.model.columns.find((c) => c.ai);
@ -2329,7 +2327,7 @@ class BaseModelSqlv2 {
if (ag) { if (ag) {
if (!response) await this.execAndParse(query); if (!response) await this.execAndParse(query);
response = await this.readByPk( response = await this.readByPk(
data[ag.title], insertObj[ag.column_name],
false, false,
{}, {},
{ ignoreView: true, getHiddenColumn: true }, { ignoreView: true, getHiddenColumn: true },
@ -2380,8 +2378,8 @@ class BaseModelSqlv2 {
} }
} else if (ai) { } else if (ai) {
const id = Array.isArray(response) const id = Array.isArray(response)
? response?.[0]?.[ai.title] ? response?.[0]?.[ai.id]
: response?.[ai.title]; : response?.[ai.id];
response = await this.readByPk( response = await this.readByPk(
id, id,
false, false,
@ -2677,9 +2675,9 @@ class BaseModelSqlv2 {
if (this.isPg || this.isMssql) { if (this.isPg || this.isMssql) {
query.returning( query.returning(
`${this.model.primaryKey.column_name} as ${this.model.primaryKey.title}`, `${this.model.primaryKey.column_name} as ${this.model.primaryKey.id}`,
); );
response = await this.execAndParse(query); response = await this.execAndParse(query, null, { raw: true });
} }
const ai = this.model.columns.find((c) => c.ai); const ai = this.model.columns.find((c) => c.ai);
@ -2691,7 +2689,7 @@ class BaseModelSqlv2 {
if (ag) { if (ag) {
if (!response) await this.execAndParse(query); if (!response) await this.execAndParse(query);
response = await this.readByPk( response = await this.readByPk(
data[ag.title], insertObj[ag.column_name],
false, false,
{}, {},
{ ignoreView: true, getHiddenColumn: true }, { ignoreView: true, getHiddenColumn: true },
@ -2749,8 +2747,8 @@ class BaseModelSqlv2 {
} }
} else if (ai) { } else if (ai) {
rowId = Array.isArray(response) rowId = Array.isArray(response)
? response?.[0]?.[ai.title] ? response?.[0]?.[ai.id]
: response?.[ai.title]; : response?.[ai.id];
} }
await Promise.all(postInsertOps.map((f) => f(rowId))); await Promise.all(postInsertOps.map((f) => f(rowId)));
@ -4223,11 +4221,11 @@ class BaseModelSqlv2 {
const groupedResult = result.reduce<Map<string | number | null, any[]>>( const groupedResult = result.reduce<Map<string | number | null, any[]>>(
(aggObj, row) => { (aggObj, row) => {
if (!aggObj.has(row[column.title])) { if (!aggObj.has(row[column.id])) {
aggObj.set(row[column.title], []); aggObj.set(row[column.id], []);
} }
aggObj.get(row[column.title]).push(row); aggObj.get(row[column.id]).push(row);
return aggObj; return aggObj;
}, },
@ -4324,11 +4322,13 @@ class BaseModelSqlv2 {
options: { options: {
skipDateConversion?: boolean; skipDateConversion?: boolean;
skipAttachmentConversion?: boolean; skipAttachmentConversion?: boolean;
skipSubstitutingColumnIds?: boolean;
raw?: boolean; // alias for skipDateConversion and skipAttachmentConversion raw?: boolean; // alias for skipDateConversion and skipAttachmentConversion
first?: boolean; first?: boolean;
} = { } = {
skipDateConversion: false, skipDateConversion: false,
skipAttachmentConversion: false, skipAttachmentConversion: false,
skipSubstitutingColumnIds: false,
raw: false, raw: false,
first: false, first: false,
}, },
@ -4336,6 +4336,7 @@ class BaseModelSqlv2 {
if (options.raw) { if (options.raw) {
options.skipDateConversion = true; options.skipDateConversion = true;
options.skipAttachmentConversion = true; options.skipAttachmentConversion = true;
options.skipSubstitutingColumnIds = true;
} }
if (options.first && typeof qb !== 'string') { if (options.first && typeof qb !== 'string') {
@ -4368,6 +4369,10 @@ class BaseModelSqlv2 {
data = this.convertDateFormat(data, childTable); data = this.convertDateFormat(data, childTable);
} }
if (!options.skipSubstitutingColumnIds) {
data = this.substituteColumnIdsWithColumnTitles(data, childTable);
}
if (options.first) { if (options.first) {
return data?.[0]; return data?.[0];
} }
@ -4375,6 +4380,35 @@ class BaseModelSqlv2 {
return data; return data;
} }
protected substituteColumnIdsWithColumnTitles(
data: Record<string, any>[],
childTable?: Model,
) {
const modelColumns = this.model?.columns.concat(childTable?.columns ?? []);
if (!modelColumns || !data.length) {
return data;
}
const idToAliasMap: Record<string, string> = {};
modelColumns.forEach((col) => {
idToAliasMap[col.id] = col.title;
});
data.forEach((item) => {
Object.entries(item).forEach(([key, value]) => {
const alias = idToAliasMap[key];
if (alias) {
item[alias] = value;
delete item[key];
}
});
});
return data;
}
protected async _convertAttachmentType( protected async _convertAttachmentType(
attachmentColumns: Record<string, any>[], attachmentColumns: Record<string, any>[],
d: Record<string, any>, d: Record<string, any>,
@ -4383,12 +4417,12 @@ class BaseModelSqlv2 {
if (d) { if (d) {
const promises = []; const promises = [];
for (const col of attachmentColumns) { for (const col of attachmentColumns) {
if (d[col.title] && typeof d[col.title] === 'string') { if (d[col.id] && typeof d[col.id] === 'string') {
d[col.title] = JSON.parse(d[col.title]); d[col.id] = JSON.parse(d[col.id]);
} }
if (d[col.title]?.length) { if (d[col.id]?.length) {
for (const attachment of d[col.title]) { for (const attachment of d[col.id]) {
// we expect array of array of attachments in case of lookup // we expect array of array of attachments in case of lookup
if (Array.isArray(attachment)) { if (Array.isArray(attachment)) {
for (const lookedUpAttachment of attachment) { for (const lookedUpAttachment of attachment) {
@ -4502,24 +4536,24 @@ class BaseModelSqlv2 {
) { ) {
if (!d) return d; if (!d) return d;
for (const col of dateTimeColumns) { for (const col of dateTimeColumns) {
if (!d[col.title]) continue; if (!d[col.id]) continue;
if (col.uidt === UITypes.Formula) { if (col.uidt === UITypes.Formula) {
if (!d[col.title] || typeof d[col.title] !== 'string') { if (!d[col.id] || typeof d[col.id] !== 'string') {
continue; continue;
} }
// remove milliseconds // remove milliseconds
if (this.isMySQL) { if (this.isMySQL) {
d[col.title] = d[col.title].replace(/\.000000/g, ''); d[col.id] = d[col.id].replace(/\.000000/g, '');
} else if (this.isMssql) { } else if (this.isMssql) {
d[col.title] = d[col.title].replace(/\.0000000 \+00:00/g, ''); d[col.id] = d[col.id].replace(/\.0000000 \+00:00/g, '');
} }
if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.title])) { if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.id])) {
// convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
// e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30 // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
d[col.title] = d[col.title].replace( d[col.id] = d[col.id].replace(
/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g,
(d: string) => { (d: string) => {
if (!dayjs(d).isValid()) return d; if (!dayjs(d).isValid()) return d;
@ -4536,7 +4570,7 @@ class BaseModelSqlv2 {
// convert all date time values to utc // convert all date time values to utc
// the datetime is either YYYY-MM-DD hh:mm:ss (xcdb) // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
// or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext) // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
d[col.title] = d[col.title].replace( d[col.id] = d[col.id].replace(
/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g, /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g,
(d: string) => { (d: string) => {
if (!dayjs(d).isValid()) { if (!dayjs(d).isValid()) {
@ -4576,15 +4610,15 @@ class BaseModelSqlv2 {
if (this.isSqlite) { if (this.isSqlite) {
if (!col.cdf) { if (!col.cdf) {
if ( if (
d[col.title].indexOf('-') === -1 && d[col.id].indexOf('-') === -1 &&
d[col.title].indexOf('+') === -1 && d[col.id].indexOf('+') === -1 &&
d[col.title].slice(-1) !== 'Z' d[col.id].slice(-1) !== 'Z'
) { ) {
// if there is no timezone info, // if there is no timezone info,
// we assume the input is on NocoDB server timezone // we assume the input is on NocoDB server timezone
// then we convert to UTC from server timezone // then we convert to UTC from server timezone
// e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00 // e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00
d[col.title] = dayjs(d[col.title]) d[col.id] = dayjs(d[col.id])
.tz(Intl.DateTimeFormat().resolvedOptions().timeZone) .tz(Intl.DateTimeFormat().resolvedOptions().timeZone)
.utc() .utc()
.format('YYYY-MM-DD HH:mm:ssZ'); .format('YYYY-MM-DD HH:mm:ssZ');
@ -4602,14 +4636,14 @@ class BaseModelSqlv2 {
keepLocalTime = false; keepLocalTime = false;
} }
if (d[col.title] instanceof Date) { if (d[col.id] instanceof Date) {
// e.g. MSSQL // e.g. MSSQL
// Wed May 10 2023 17:47:46 GMT+0800 (Hong Kong Standard Time) // Wed May 10 2023 17:47:46 GMT+0800 (Hong Kong Standard Time)
keepLocalTime = false; keepLocalTime = false;
} }
// e.g. 01.01.2022 10:00:00+05:30 -> 2022-01-01 04:30:00+00:00 // e.g. 01.01.2022 10:00:00+05:30 -> 2022-01-01 04:30:00+00:00
// e.g. 2023-05-09 11:41:49 -> 2023-05-09 11:41:49+00:00 // e.g. 2023-05-09 11:41:49 -> 2023-05-09 11:41:49+00:00
d[col.title] = dayjs(d[col.title]) d[col.id] = dayjs(d[col.id])
// keep the local time // keep the local time
.utc(keepLocalTime) .utc(keepLocalTime)
// show the timezone even for Mysql // show the timezone even for Mysql
@ -5413,7 +5447,9 @@ export function _wherePk(primaryKeys: Column[], id: unknown | unknown[]) {
if (id && typeof id === 'object' && !Array.isArray(id)) { if (id && typeof id === 'object' && !Array.isArray(id)) {
// verify all pk columns are present in id object // verify all pk columns are present in id object
for (const pk of primaryKeys) { for (const pk of primaryKeys) {
if (pk.title in id) { if (pk.id in id) {
where[pk.column_name] = id[pk.id];
} else if (pk.title in id) {
where[pk.column_name] = id[pk.title]; where[pk.column_name] = id[pk.title];
} else if (pk.column_name in id) { } else if (pk.column_name in id) {
where[pk.column_name] = id[pk.column_name]; where[pk.column_name] = id[pk.column_name];

Loading…
Cancel
Save