Browse Source

refactor: list api optimisation and skipping strict type check in test

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/6445/head
Pranav C 1 year ago
parent
commit
672173d2d7
  1. 18
      packages/nc-gui/assets/nc-icons/cpu.svg
  2. 5
      packages/nc-gui/assets/nc-icons/database1.svg
  3. 14
      packages/nc-gui/assets/nc-icons/server.svg
  4. 3
      packages/nc-gui/components/general/ApiTiming.vue
  5. 2
      packages/nc-gui/components/smartsheet/Gallery.vue
  6. 25
      packages/nc-gui/components/smartsheet/Pagination.vue
  7. 1
      packages/nc-gui/components/smartsheet/grid/Table.vue
  8. 10
      packages/nocodb/src/controllers/data-alias.controller.ts
  9. 344
      packages/nocodb/src/db/BaseModelSqlv2.ts
  10. 33
      packages/nocodb/src/models/Base.ts
  11. 35
      packages/nocodb/src/services/datas.service.ts
  12. 4
      tests/playwright/tests/db/features/filters.spec.ts
  13. 2
      tests/playwright/tests/utils/general.ts

18
packages/nc-gui/assets/nc-icons/cpu.svg

@ -0,0 +1,18 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_664_57284)">
<path d="M13.3334 7.5V3.99984C13.3334 3.26346 12.7365 2.6665 12.0001 2.6665H4.00008C3.2637 2.6665 2.66675 3.26346 2.66675 3.99984V11.9998C2.66675 12.7362 3.2637 13.3332 4.00008 13.3332H7.5" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M10 6H6V10" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M6 0.666504V2.6665" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M10 0.666504V2.6665" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M6 13.3335V15.3335" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M13.3333 6H15.3333" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M0.666748 6H2.66675" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M0.666748 9.3335H2.66675" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15ZM12 12.5C11.7239 12.5 11.5 12.2761 11.5 12V10.5C11.5 10.2239 11.7239 10 12 10C12.2761 10 12.5 10.2239 12.5 10.5V11.5H13.5C13.7761 11.5 14 11.7239 14 12C14 12.2761 13.7761 12.5 13.5 12.5H12Z" stroke="transparent" fill="currentColor"/>
</g>
<defs>
<clipPath id="clip0_664_57284">
<rect width="16" height="16" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 1.7 KiB

5
packages/nc-gui/assets/nc-icons/database1.svg

@ -0,0 +1,5 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M8 5.3335C11.3137 5.3335 14 4.43807 14 3.3335C14 2.22893 11.3137 1.3335 8 1.3335C4.68629 1.3335 2 2.22893 2 3.3335C2 4.43807 4.68629 5.3335 8 5.3335Z" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M2 3.3335V12.6668C2 13.7735 5 14.5 7.5 14.5M14 3.3335V7.5M2 8.16683C2 9.2735 5.5 10 8 10" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15ZM12 12.5C11.7239 12.5 11.5 12.2761 11.5 12V10.5C11.5 10.2239 11.7239 10 12 10C12.2761 10 12.5 10.2239 12.5 10.5V11.5H13.5C13.7761 11.5 14 11.7239 14 12C14 12.2761 13.7761 12.5 13.5 12.5H12Z" stroke="transparent" fill="currentColor"/>
</svg>

After

Width:  |  Height:  |  Size: 946 B

14
packages/nc-gui/assets/nc-icons/server.svg

@ -0,0 +1,14 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_664_57268)">
<path d="M7.5 9.3335H2.66659C1.93021 9.3335 1.33325 9.93045 1.33325 10.6668V13.3335C1.33325 14.0699 1.93021 14.6668 2.66659 14.6668H7.5" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M4 12H4.00698" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M13.3333 1.3335H2.66659C1.93021 1.3335 1.33325 1.93045 1.33325 2.66683V5.3335C1.33325 6.06988 1.93021 6.66683 2.66659 6.66683H13.3333C14.0696 6.66683 14.6666 6.06988 14.6666 5.3335V2.66683C14.6666 1.93045 14.0696 1.3335 13.3333 1.3335Z" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M4 4H4.00698" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15ZM12 12.5C11.7239 12.5 11.5 12.2761 11.5 12V10.5C11.5 10.2239 11.7239 10 12 10C12.2761 10 12.5 10.2239 12.5 10.5V11.5H13.5C13.7761 11.5 14 11.7239 14 12C14 12.2761 13.7761 12.5 13.5 12.5H12Z" stroke="transparent" fill="currentColor"/>
</g>
<defs>
<clipPath id="clip0_664_57268">
<rect width="16" height="16" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 1.4 KiB

3
packages/nc-gui/components/general/ApiTiming.vue

@ -0,0 +1,3 @@
<template>
<span></span>
</template>

2
packages/nc-gui/components/smartsheet/Gallery.vue

@ -359,7 +359,7 @@ watch(
</div>
</a-dropdown>
<LazySmartsheetPagination v-model:pagination-data="paginationData" :change-page="changePage" />
<LazySmartsheetPagination v-model:pagination-data="paginationData" show-api-timing :change-page="changePage" />
<Suspense>
<LazySmartsheetExpandedForm
v-if="expandedFormRow && expandedFormDlg"

25
packages/nc-gui/components/smartsheet/Pagination.vue

@ -12,6 +12,7 @@ interface Props {
customLabel?: string
fixedSize?: number
extraStyle?: string
showApiTiming?: boolean
}
const props = defineProps<Props>()
@ -61,8 +62,8 @@ const isRTLLanguage = computed(() => isRtlLang(locale.value as keyof typeof Lang
isGroupBy ? 'margin-top:1px; border-radius: 0 0 12px 12px !important;' : ''
}${extraStyle}`"
>
<slot name="add-record" />
<div class="flex-1">
<div class="flex-1 flex items-center">
<slot name="add-record" />
<span
v-if="!alignCountOnRight && count !== null && count !== Infinity"
class="caption ml-2.5 text-gray-500 text-xs"
@ -93,15 +94,17 @@ const isRTLLanguage = computed(() => isRtlLang(locale.value as keyof typeof Lang
</a-input>
</div>
</template>
<div class="flex-1 text-right">
<span
v-if="alignCountOnRight && count !== null && count !== Infinity"
class="caption nc-grid-row-count mr-2.5 text-gray-500 text-xs"
data-testid="grid-pagination"
>
{{ count }} {{ customLabel ? customLabel : count !== 1 ? $t('objects.records') : $t('objects.record') }}
</span>
<div class="flex-1 flex justify-end items-center">
<GeneralApiTiming v-if="isEeUI && props.showApiTiming" class="m-1" />
<div class="text-right">
<span
v-if="alignCountOnRight && count !== null && count !== Infinity"
class="caption nc-grid-row-count mr-2.5 text-gray-500 text-xs"
data-testid="grid-pagination"
>
{{ count }} {{ customLabel ? customLabel : count !== 1 ? $t('objects.records') : $t('objects.record') }}
</span>
</div>
</div>
</div>
</template>

1
packages/nc-gui/components/smartsheet/grid/Table.vue

@ -1575,6 +1575,7 @@ const expandAndLooseFocus = (row: Row, col: Record<string, any>) => {
<LazySmartsheetPagination
v-else-if="headerOnly !== true"
v-model:pagination-data="paginationDataRef"
show-api-timing
align-count-on-right
:change-page="changePage"
:hide-sidebars="paginationStyleRef?.hideSidebars === true"

10
packages/nocodb/src/controllers/data-alias.controller.ts

@ -44,8 +44,14 @@ export class DataAliasController {
viewName: viewName,
disableOptimization: opt === 'false',
});
const elapsedSeconds = parseHrtimeToMilliSeconds(process.hrtime(startTime));
res.setHeader('xc-db-response', elapsedSeconds);
const elapsedMilliSeconds = parseHrtimeToMilliSeconds(
process.hrtime(startTime),
);
res.setHeader('xc-db-response', elapsedMilliSeconds);
if (responseData['stats']) {
responseData['stats'].apiHandlingTime = elapsedMilliSeconds;
}
res.json(responseData);
}

344
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -1509,194 +1509,196 @@ class BaseModelSqlv2 {
const proto: any = { __columnAliases: {} };
const columns = await this.model.getColumns();
for (const column of columns) {
switch (column.uidt) {
case UITypes.Lookup:
{
// @ts-ignore
const colOptions: LookupColumn = await column.getColOptions();
const relCol = await Column.get({
colId: colOptions.fk_relation_column_id,
});
const relColTitle =
relCol.uidt === UITypes.Links
? `_nc_lk_${relCol.title}`
: relCol.title;
proto.__columnAliases[column.title] = {
path: [
relColTitle,
(await Column.get({ colId: colOptions.fk_lookup_column_id }))
?.title,
],
};
}
break;
case UITypes.Links:
case UITypes.LinkToAnotherRecord:
{
this._columns[column.title] = column;
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
// const parentColumn = await colOptions.getParentColumn();
if (colOptions?.type === 'hm') {
const listLoader = new DataLoader(async (ids: string[]) => {
if (ids.length > 1) {
const data = await this.multipleHmList(
{
colId: column.id,
ids,
},
(listLoader as any).args,
);
return ids.map((id: string) => (data[id] ? data[id] : []));
} else {
return [
await this.hmList(
await Promise.all(
columns.map(async (column) => {
switch (column.uidt) {
case UITypes.Lookup:
{
// @ts-ignore
const colOptions: LookupColumn = await column.getColOptions();
const relCol = await Column.get({
colId: colOptions.fk_relation_column_id,
});
const relColTitle =
relCol.uidt === UITypes.Links
? `_nc_lk_${relCol.title}`
: relCol.title;
proto.__columnAliases[column.title] = {
path: [
relColTitle,
(await Column.get({ colId: colOptions.fk_lookup_column_id }))
?.title,
],
};
}
break;
case UITypes.Links:
case UITypes.LinkToAnotherRecord:
{
this._columns[column.title] = column;
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
// const parentColumn = await colOptions.getParentColumn();
if (colOptions?.type === 'hm') {
const listLoader = new DataLoader(async (ids: string[]) => {
if (ids.length > 1) {
const data = await this.multipleHmList(
{
colId: column.id,
id: ids[0],
ids,
},
(listLoader as any).args,
),
];
}
});
const self: BaseModelSqlv2 = this;
proto[
column.uidt === UITypes.Links
? `_nc_lk_${column.title}`
: column.title
] = async function (args): Promise<any> {
(listLoader as any).args = args;
return listLoader.load(
getCompositePk(self.model.primaryKeys, this),
);
};
// defining HasMany count method within GQL Type class
// Object.defineProperty(type.prototype, column.alias, {
// async value(): Promise<any> {
// return listLoader.load(this[model.pk.alias]);
// },
// configurable: true
// });
} else if (colOptions.type === 'mm') {
const listLoader = new DataLoader(async (ids: string[]) => {
if (ids?.length > 1) {
const data = await this.multipleMmList(
{
parentIds: ids,
colId: column.id,
},
(listLoader as any).args,
);
return ids.map((id: string) => (data[id] ? data[id] : []));
} else {
return [
await this.hmList(
{
colId: column.id,
id: ids[0],
},
(listLoader as any).args,
),
];
}
});
const self: BaseModelSqlv2 = this;
proto[
column.uidt === UITypes.Links
? `_nc_lk_${column.title}`
: column.title
] = async function (args): Promise<any> {
(listLoader as any).args = args;
return listLoader.load(
getCompositePk(self.model.primaryKeys, this),
);
return data;
} else {
return [
await this.mmList(
};
// defining HasMany count method within GQL Type class
// Object.defineProperty(type.prototype, column.alias, {
// async value(): Promise<any> {
// return listLoader.load(this[model.pk.alias]);
// },
// configurable: true
// });
} else if (colOptions.type === 'mm') {
const listLoader = new DataLoader(async (ids: string[]) => {
if (ids?.length > 1) {
const data = await this.multipleMmList(
{
parentId: ids[0],
parentIds: ids,
colId: column.id,
},
(listLoader as any).args,
),
];
}
});
);
const self: BaseModelSqlv2 = this;
// const childColumn = await colOptions.getChildColumn();
proto[
column.uidt === UITypes.Links
? `_nc_lk_${column.title}`
: column.title
] = async function (args): Promise<any> {
(listLoader as any).args = args;
return await listLoader.load(
getCompositePk(self.model.primaryKeys, this),
);
};
} else if (colOptions.type === 'bt') {
// @ts-ignore
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
const pCol = await Column.get({
colId: colOptions.fk_parent_column_id,
});
const cCol = await Column.get({
colId: colOptions.fk_child_column_id,
});
const readLoader = new DataLoader(async (_ids: string[]) => {
// handle binary(16) foreign keys
const ids = _ids.map((id) => {
if (pCol.ct !== 'binary(16)') return id;
//Cast the id to string.
const idAsString = id + '';
// Check if the id is a UUID and the column is binary(16)
const isUUIDBinary16 =
idAsString.length === 36 || idAsString.length === 32;
// If the id is a UUID and the column is binary(16), convert the id to a Buffer. Otherwise, return null to indicate that the id is not a UUID.
const idAsUUID = isUUIDBinary16
? idAsString.length === 32
? idAsString.replace(
/(.{8})(.{4})(.{4})(.{4})(.{12})/,
'$1-$2-$3-$4-$5',
)
: idAsString
: null;
return idAsUUID
? Buffer.from(idAsUUID.replace(/-/g, ''), 'hex')
: id;
return data;
} else {
return [
await this.mmList(
{
parentId: ids[0],
colId: column.id,
},
(listLoader as any).args,
),
];
}
});
const data = await (
await Model.getBaseModelSQL({
id: pCol.fk_model_id,
dbDriver: this.dbDriver,
})
).list(
{
fieldsSet: (readLoader as any).args?.fieldsSet,
filterArr: [
new Filter({
id: null,
fk_column_id: pCol.id,
fk_model_id: pCol.fk_model_id,
value: ids as any[],
comparison_op: 'in',
}),
],
},
true,
);
const self: BaseModelSqlv2 = this;
// const childColumn = await colOptions.getChildColumn();
proto[
column.uidt === UITypes.Links
? `_nc_lk_${column.title}`
: column.title
] = async function (args): Promise<any> {
(listLoader as any).args = args;
return await listLoader.load(
getCompositePk(self.model.primaryKeys, this),
);
};
} else if (colOptions.type === 'bt') {
// @ts-ignore
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
const pCol = await Column.get({
colId: colOptions.fk_parent_column_id,
});
const cCol = await Column.get({
colId: colOptions.fk_child_column_id,
});
const readLoader = new DataLoader(async (_ids: string[]) => {
// handle binary(16) foreign keys
const ids = _ids.map((id) => {
if (pCol.ct !== 'binary(16)') return id;
//Cast the id to string.
const idAsString = id + '';
// Check if the id is a UUID and the column is binary(16)
const isUUIDBinary16 =
idAsString.length === 36 || idAsString.length === 32;
// If the id is a UUID and the column is binary(16), convert the id to a Buffer. Otherwise, return null to indicate that the id is not a UUID.
const idAsUUID = isUUIDBinary16
? idAsString.length === 32
? idAsString.replace(
/(.{8})(.{4})(.{4})(.{4})(.{12})/,
'$1-$2-$3-$4-$5',
)
: idAsString
: null;
return idAsUUID
? Buffer.from(idAsUUID.replace(/-/g, ''), 'hex')
: id;
});
const data = await (
await Model.getBaseModelSQL({
id: pCol.fk_model_id,
dbDriver: this.dbDriver,
})
).list(
{
fieldsSet: (readLoader as any).args?.fieldsSet,
filterArr: [
new Filter({
id: null,
fk_column_id: pCol.id,
fk_model_id: pCol.fk_model_id,
value: ids as any[],
comparison_op: 'in',
}),
],
},
true,
);
const gs = groupBy(data, pCol.title);
return _ids.map(async (id: string) => gs?.[id]?.[0]);
});
const gs = groupBy(data, pCol.title);
return _ids.map(async (id: string) => gs?.[id]?.[0]);
});
// defining HasMany count method within GQL Type class
proto[column.title] = async function (args?: any) {
if (
this?.[cCol?.title] === null ||
this?.[cCol?.title] === undefined
)
return null;
// defining HasMany count method within GQL Type class
proto[column.title] = async function (args?: any) {
if (
this?.[cCol?.title] === null ||
this?.[cCol?.title] === undefined
)
return null;
(readLoader as any).args = args;
(readLoader as any).args = args;
return await readLoader.load(this?.[cCol?.title]);
};
// todo : handle mm
return await readLoader.load(this?.[cCol?.title]);
};
// todo : handle mm
}
}
}
break;
}
}
break;
}
}),
);
this._proto = proto;
return proto;
}

33
packages/nocodb/src/models/Base.ts

@ -17,6 +17,7 @@ import Noco from '~/Noco';
import { extractProps } from '~/helpers/extractProps';
import { NcError } from '~/helpers/catchError';
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import { parseMetaProp, stringifyMetaProp } from '~/utils/modelUtils';
// todo: hide credentials
export default class Base implements BaseType {
@ -31,6 +32,7 @@ export default class Base implements BaseType {
order?: number;
erd_uuid?: string;
enabled?: BoolType;
meta?: any;
constructor(base: Partial<Base>) {
Object.assign(this, base);
@ -41,7 +43,12 @@ export default class Base implements BaseType {
}
public static async createBase(
base: BaseType & { projectId: string; created_at?; updated_at? },
base: BaseType & {
projectId: string;
created_at?;
updated_at?;
meta?: any;
},
ncMeta = Noco.ncMeta,
) {
const insertObj = extractProps(base, [
@ -54,12 +61,18 @@ export default class Base implements BaseType {
'inflection_table',
'order',
'enabled',
'meta',
]);
insertObj.config = CryptoJS.AES.encrypt(
JSON.stringify(base.config),
Noco.getConfig()?.auth?.jwt?.secret,
).toString();
if ('meta' in insertObj) {
insertObj.meta = stringifyMetaProp(insertObj);
}
const { id } = await ncMeta.metaInsert2(
base.projectId,
null,
@ -84,9 +97,9 @@ export default class Base implements BaseType {
public static async updateBase(
baseId: string,
base: BaseType & {
id: string;
projectId: string;
skipReorder?: boolean;
meta?: any;
},
ncMeta = Noco.ncMeta,
) {
@ -109,6 +122,7 @@ export default class Base implements BaseType {
'inflection_table',
'order',
'enabled',
'meta',
]);
if (updateObj.config) {
@ -118,6 +132,10 @@ export default class Base implements BaseType {
).toString();
}
if ('meta' in updateObj) {
updateObj.meta = stringifyMetaProp(updateObj);
}
// type property is undefined even if not provided
if (!updateObj.type) {
updateObj.type = oldBase.type;
@ -166,6 +184,12 @@ export default class Base implements BaseType {
},
},
);
// parse JSON metadata
for (const base of baseDataList) {
base.meta = parseMetaProp(base, 'meta');
}
await NocoCache.setList(CacheScope.BASE, [args.projectId], baseDataList);
}
@ -185,6 +209,11 @@ export default class Base implements BaseType {
));
if (!baseData) {
baseData = await ncMeta.metaGet2(null, null, MetaTable.BASES, id);
if (baseData) {
baseData.meta = parseMetaProp(baseData, 'meta');
}
await NocoCache.set(`${CacheScope.BASE}:${id}`, baseData);
}
return this.castType(baseData);

35
packages/nocodb/src/services/datas.service.ts

@ -159,21 +159,26 @@ export class DatasService {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
let data = [];
let count = 0;
try {
data = await nocoExecute(
ast,
await baseModel.list(listArgs),
{},
listArgs,
);
count = await baseModel.count(listArgs);
} catch (e) {
console.log(e);
NcError.internalServerError('Please check server log for more details');
}
const [count, data] = await Promise.all([
baseModel.count(listArgs),
(async () => {
let data = [];
try {
data = await nocoExecute(
ast,
await baseModel.list(listArgs),
{},
listArgs,
);
} catch (e) {
console.log(e);
NcError.internalServerError(
'Please check server log for more details',
);
}
return data;
})(),
]);
return new PagedResponseImpl(data, {
...query,
count,

4
tests/playwright/tests/db/features/filters.spec.ts

@ -856,14 +856,14 @@ test.describe('Filter Tests: AddOn', () => {
op: 'is checked',
value: null,
rowCount: records.list.filter(r => {
return r[dataType] === (context.dbType === 'pg' ? true : 1);
return r[dataType];
}).length,
},
{
op: 'is not checked',
value: null,
rowCount: records.list.filter(r => {
return r[dataType] !== (context.dbType === 'pg' ? true : 1);
return r[dataType];
}).length,
},
];

2
tests/playwright/tests/utils/general.ts

@ -38,7 +38,7 @@ function isSubset(obj, potentialSubset) {
return false;
}
// eslint-disable-next-line no-prototype-builtins
} else if (!obj.hasOwnProperty(prop) || objValue !== potentialValue) {
} else if (!obj.hasOwnProperty(prop) || objValue != potentialValue) {
return false;
}
}

Loading…
Cancel
Save