Browse Source

Merge pull request #6445 from nocodb/nc-fix/bt-performance-issue

fix: Belongs to performance issue
pull/6463/head
Pranav C 1 year ago committed by GitHub
parent
commit
eea5a9405a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 18
      packages/nc-gui/assets/nc-icons/cpu.svg
  2. 5
      packages/nc-gui/assets/nc-icons/database1.svg
  3. 14
      packages/nc-gui/assets/nc-icons/server.svg
  4. 3
      packages/nc-gui/components/general/ApiTiming.vue
  5. 2
      packages/nc-gui/components/smartsheet/Gallery.vue
  6. 11
      packages/nc-gui/components/smartsheet/Pagination.vue
  7. 1
      packages/nc-gui/components/smartsheet/grid/Table.vue
  8. 10
      packages/nocodb/src/controllers/data-alias.controller.ts
  9. 69
      packages/nocodb/src/db/BaseModelSqlv2.ts
  10. 4
      packages/nocodb/src/helpers/getAst.ts
  11. 33
      packages/nocodb/src/models/Base.ts
  12. 2
      packages/nocodb/src/modules/datas/helpers.ts
  13. 13
      packages/nocodb/src/services/datas.service.ts
  14. 4
      tests/playwright/tests/db/features/filters.spec.ts
  15. 4
      tests/playwright/tests/utils/general.ts

18
packages/nc-gui/assets/nc-icons/cpu.svg

@ -0,0 +1,18 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_664_57284)">
<path d="M13.3334 7.5V3.99984C13.3334 3.26346 12.7365 2.6665 12.0001 2.6665H4.00008C3.2637 2.6665 2.66675 3.26346 2.66675 3.99984V11.9998C2.66675 12.7362 3.2637 13.3332 4.00008 13.3332H7.5" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M10 6H6V10" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M6 0.666504V2.6665" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M10 0.666504V2.6665" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M6 13.3335V15.3335" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M13.3333 6H15.3333" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M0.666748 6H2.66675" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M0.666748 9.3335H2.66675" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15ZM12 12.5C11.7239 12.5 11.5 12.2761 11.5 12V10.5C11.5 10.2239 11.7239 10 12 10C12.2761 10 12.5 10.2239 12.5 10.5V11.5H13.5C13.7761 11.5 14 11.7239 14 12C14 12.2761 13.7761 12.5 13.5 12.5H12Z" stroke="transparent" fill="currentColor"/>
</g>
<defs>
<clipPath id="clip0_664_57284">
<rect width="16" height="16" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 1.7 KiB

5
packages/nc-gui/assets/nc-icons/database1.svg

@ -0,0 +1,5 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M8 5.3335C11.3137 5.3335 14 4.43807 14 3.3335C14 2.22893 11.3137 1.3335 8 1.3335C4.68629 1.3335 2 2.22893 2 3.3335C2 4.43807 4.68629 5.3335 8 5.3335Z" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M2 3.3335V12.6668C2 13.7735 5 14.5 7.5 14.5M14 3.3335V7.5M2 8.16683C2 9.2735 5.5 10 8 10" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15ZM12 12.5C11.7239 12.5 11.5 12.2761 11.5 12V10.5C11.5 10.2239 11.7239 10 12 10C12.2761 10 12.5 10.2239 12.5 10.5V11.5H13.5C13.7761 11.5 14 11.7239 14 12C14 12.2761 13.7761 12.5 13.5 12.5H12Z" stroke="transparent" fill="currentColor"/>
</svg>

After

Width:  |  Height:  |  Size: 946 B

14
packages/nc-gui/assets/nc-icons/server.svg

@ -0,0 +1,14 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_664_57268)">
<path d="M7.5 9.3335H2.66659C1.93021 9.3335 1.33325 9.93045 1.33325 10.6668V13.3335C1.33325 14.0699 1.93021 14.6668 2.66659 14.6668H7.5" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M4 12H4.00698" stroke="currentColor" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M13.3333 1.3335H2.66659C1.93021 1.3335 1.33325 1.93045 1.33325 2.66683V5.3335C1.33325 6.06988 1.93021 6.66683 2.66659 6.66683H13.3333C14.0696 6.66683 14.6666 6.06988 14.6666 5.3335V2.66683C14.6666 1.93045 14.0696 1.3335 13.3333 1.3335Z" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M4 4H4.00698" stroke="currentColor" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15ZM12 12.5C11.7239 12.5 11.5 12.2761 11.5 12V10.5C11.5 10.2239 11.7239 10 12 10C12.2761 10 12.5 10.2239 12.5 10.5V11.5H13.5C13.7761 11.5 14 11.7239 14 12C14 12.2761 13.7761 12.5 13.5 12.5H12Z" stroke="transparent" fill="currentColor"/>
</g>
<defs>
<clipPath id="clip0_664_57268">
<rect width="16" height="16" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 1.4 KiB

3
packages/nc-gui/components/general/ApiTiming.vue

@ -0,0 +1,3 @@
<template>
<span></span>
</template>

2
packages/nc-gui/components/smartsheet/Gallery.vue

@ -359,7 +359,7 @@ watch(
</div>
</a-dropdown>
<LazySmartsheetPagination v-model:pagination-data="paginationData" :change-page="changePage" />
<LazySmartsheetPagination v-model:pagination-data="paginationData" show-api-timing :change-page="changePage" />
<Suspense>
<LazySmartsheetExpandedForm
v-if="expandedFormRow && expandedFormDlg"

11
packages/nc-gui/components/smartsheet/Pagination.vue

@ -12,6 +12,7 @@ interface Props {
customLabel?: string
fixedSize?: number
extraStyle?: string
showApiTiming?: boolean
}
const props = defineProps<Props>()
@ -61,8 +62,8 @@ const isRTLLanguage = computed(() => isRtlLang(locale.value as keyof typeof Lang
isGroupBy ? 'margin-top:1px; border-radius: 0 0 12px 12px !important;' : ''
}${extraStyle}`"
>
<div class="flex-1 flex items-center">
<slot name="add-record" />
<div class="flex-1">
<span
v-if="!alignCountOnRight && count !== null && count !== Infinity"
class="caption ml-2.5 text-gray-500 text-xs"
@ -93,10 +94,11 @@ const isRTLLanguage = computed(() => isRtlLang(locale.value as keyof typeof Lang
</a-input>
</div>
</template>
<div class="flex-1 text-right">
<div class="flex-1 flex justify-end items-center">
<GeneralApiTiming v-if="isEeUI && props.showApiTiming" class="m-1" />
<div class="text-right">
<span
v-if="alignCountOnRight && count !== null && count !== Infinity"
v-if="alignCountOnRight && count !== Infinity"
class="caption nc-grid-row-count mr-2.5 text-gray-500 text-xs"
data-testid="grid-pagination"
>
@ -104,6 +106,7 @@ const isRTLLanguage = computed(() => isRtlLang(locale.value as keyof typeof Lang
</span>
</div>
</div>
</div>
</template>
<style lang="scss">

1
packages/nc-gui/components/smartsheet/grid/Table.vue

@ -1575,6 +1575,7 @@ const expandAndLooseFocus = (row: Row, col: Record<string, any>) => {
<LazySmartsheetPagination
v-else-if="headerOnly !== true"
v-model:pagination-data="paginationDataRef"
show-api-timing
align-count-on-right
:change-page="changePage"
:hide-sidebars="paginationStyleRef?.hideSidebars === true"

10
packages/nocodb/src/controllers/data-alias.controller.ts

@ -44,8 +44,14 @@ export class DataAliasController {
viewName: viewName,
disableOptimization: opt === 'false',
});
const elapsedSeconds = parseHrtimeToMilliSeconds(process.hrtime(startTime));
res.setHeader('xc-db-response', elapsedSeconds);
const elapsedMilliSeconds = parseHrtimeToMilliSeconds(
process.hrtime(startTime),
);
res.setHeader('xc-db-response', elapsedMilliSeconds);
if (responseData['stats']) {
responseData['stats'].apiHandlingTime = elapsedMilliSeconds;
}
res.json(responseData);
}

69
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -134,7 +134,7 @@ class BaseModelSqlv2 {
): Promise<any> {
const qb = this.dbDriver(this.tnPath);
const { ast, dependencyFields } = await getAst({
const { ast, dependencyFields, parsedQuery } = await getAst({
query,
model: this.model,
view: ignoreView ? null : this.viewId && (await View.get(this.viewId)),
@ -165,7 +165,7 @@ class BaseModelSqlv2 {
data.__proto__ = proto;
}
return data ? await nocoExecute(ast, data, {}, dependencyFields) : null;
return data ? await nocoExecute(ast, data, {}, parsedQuery) : null;
}
public async exist(id?: any): Promise<any> {
@ -1509,7 +1509,8 @@ class BaseModelSqlv2 {
const proto: any = { __columnAliases: {} };
const columns = await this.model.getColumns();
for (const column of columns) {
await Promise.all(
columns.map(async (column) => {
switch (column.uidt) {
case UITypes.Lookup:
{
@ -1537,7 +1538,6 @@ class BaseModelSqlv2 {
this._columns[column.title] = column;
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
// const parentColumn = await colOptions.getParentColumn();
if (colOptions?.type === 'hm') {
const listLoader = new DataLoader(async (ids: string[]) => {
@ -1574,14 +1574,6 @@ class BaseModelSqlv2 {
getCompositePk(self.model.primaryKeys, this),
);
};
// defining HasMany count method within GQL Type class
// Object.defineProperty(type.prototype, column.alias, {
// async value(): Promise<any> {
// return listLoader.load(this[model.pk.alias]);
// },
// configurable: true
// });
} else if (colOptions.type === 'mm') {
const listLoader = new DataLoader(async (ids: string[]) => {
if (ids?.length > 1) {
@ -1608,7 +1600,6 @@ class BaseModelSqlv2 {
});
const self: BaseModelSqlv2 = this;
// const childColumn = await colOptions.getChildColumn();
proto[
column.uidt === UITypes.Links
? `_nc_lk_${column.title}`
@ -1629,7 +1620,36 @@ class BaseModelSqlv2 {
const cCol = await Column.get({
colId: colOptions.fk_child_column_id,
});
const readLoader = new DataLoader(async (ids: string[]) => {
// use dataloader to get batches of parent data together rather than getting them individually
// it takes individual keys and callback is invoked with an array of values and we can get the
// result for all those together and return the value in the same order as in the array
// this way all parents data extracted together
const readLoader = new DataLoader(async (_ids: string[]) => {
// handle binary(16) foreign keys
const ids = _ids.map((id) => {
if (pCol.ct !== 'binary(16)') return id;
// Cast the id to string.
const idAsString = id + '';
// Check if the id is a UUID and the column is binary(16)
const isUUIDBinary16 =
idAsString.length === 36 || idAsString.length === 32;
// If the id is a UUID and the column is binary(16), convert the id to a Buffer. Otherwise, return null to indicate that the id is not a UUID.
const idAsUUID = isUUIDBinary16
? idAsString.length === 32
? idAsString.replace(
/(.{8})(.{4})(.{4})(.{4})(.{12})/,
'$1-$2-$3-$4-$5',
)
: idAsString
: null;
return idAsUUID
? Buffer.from(idAsUUID.replace(/-/g, ''), 'hex')
: id;
});
const data = await (
await Model.getBaseModelSQL({
id: pCol.fk_model_id,
@ -1637,17 +1657,25 @@ class BaseModelSqlv2 {
})
).list(
{
// limit: ids.length,
where: `(${pCol.column_name},in,${ids.join(',')})`,
fieldsSet: (readLoader as any).args?.fieldsSet,
filterArr: [
new Filter({
id: null,
fk_column_id: pCol.id,
fk_model_id: pCol.fk_model_id,
value: ids as any[],
comparison_op: 'in',
}),
],
},
true,
);
const gs = groupBy(data, pCol.title);
return ids.map(async (id: string) => gs?.[id]?.[0]);
const groupedList = groupBy(data, pCol.title);
return _ids.map(async (id: string) => groupedList?.[id]?.[0]);
});
// defining HasMany count method within GQL Type class
// defining BelongsTo read resolver method
proto[column.title] = async function (args?: any) {
if (
this?.[cCol?.title] === null ||
@ -1664,7 +1692,8 @@ class BaseModelSqlv2 {
}
break;
}
}
}),
);
this._proto = proto;
return proto;
}

4
packages/nocodb/src/helpers/getAst.ts

@ -56,7 +56,7 @@ const getAst = async ({
await extractDependencies(model.displayValue, dependencyFields);
return { ast, dependencyFields };
return { ast, dependencyFields, parsedQuery: dependencyFields };
}
let fields = query?.fields || query?.f;
@ -154,7 +154,7 @@ const getAst = async ({
};
}, Promise.resolve({}));
return { ast, dependencyFields };
return { ast, dependencyFields, parsedQuery: dependencyFields };
};
const extractDependencies = async (

33
packages/nocodb/src/models/Base.ts

@ -17,6 +17,7 @@ import Noco from '~/Noco';
import { extractProps } from '~/helpers/extractProps';
import { NcError } from '~/helpers/catchError';
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import { parseMetaProp, stringifyMetaProp } from '~/utils/modelUtils';
// todo: hide credentials
export default class Base implements BaseType {
@ -31,6 +32,7 @@ export default class Base implements BaseType {
order?: number;
erd_uuid?: string;
enabled?: BoolType;
meta?: any;
constructor(base: Partial<Base>) {
Object.assign(this, base);
@ -41,7 +43,12 @@ export default class Base implements BaseType {
}
public static async createBase(
base: BaseType & { projectId: string; created_at?; updated_at? },
base: BaseType & {
projectId: string;
created_at?;
updated_at?;
meta?: any;
},
ncMeta = Noco.ncMeta,
) {
const insertObj = extractProps(base, [
@ -54,12 +61,18 @@ export default class Base implements BaseType {
'inflection_table',
'order',
'enabled',
'meta',
]);
insertObj.config = CryptoJS.AES.encrypt(
JSON.stringify(base.config),
Noco.getConfig()?.auth?.jwt?.secret,
).toString();
if ('meta' in insertObj) {
insertObj.meta = stringifyMetaProp(insertObj);
}
const { id } = await ncMeta.metaInsert2(
base.projectId,
null,
@ -84,9 +97,9 @@ export default class Base implements BaseType {
public static async updateBase(
baseId: string,
base: BaseType & {
id: string;
projectId: string;
skipReorder?: boolean;
meta?: any;
},
ncMeta = Noco.ncMeta,
) {
@ -109,6 +122,7 @@ export default class Base implements BaseType {
'inflection_table',
'order',
'enabled',
'meta',
]);
if (updateObj.config) {
@ -118,6 +132,10 @@ export default class Base implements BaseType {
).toString();
}
if ('meta' in updateObj) {
updateObj.meta = stringifyMetaProp(updateObj);
}
// type property is undefined even if not provided
if (!updateObj.type) {
updateObj.type = oldBase.type;
@ -166,6 +184,12 @@ export default class Base implements BaseType {
},
},
);
// parse JSON metadata
for (const base of baseDataList) {
base.meta = parseMetaProp(base, 'meta');
}
await NocoCache.setList(CacheScope.BASE, [args.projectId], baseDataList);
}
@ -185,6 +209,11 @@ export default class Base implements BaseType {
));
if (!baseData) {
baseData = await ncMeta.metaGet2(null, null, MetaTable.BASES, id);
if (baseData) {
baseData.meta = parseMetaProp(baseData, 'meta');
}
await NocoCache.set(`${CacheScope.BASE}:${id}`, baseData);
}
return this.castType(baseData);

2
packages/nocodb/src/modules/datas/helpers.ts

@ -262,7 +262,7 @@ export async function getDbRows(param: {
ast,
await baseModel.list({ ...listArgs, ...dependencyFields, offset, limit }),
{},
query,
dependencyFields,
);
if (!rows?.length) {

13
packages/nocodb/src/services/datas.service.ts

@ -159,8 +159,10 @@ export class DatasService {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
const [count, data] = await Promise.all([
baseModel.count(listArgs),
(async () => {
let data = [];
let count = 0;
try {
data = await nocoExecute(
ast,
@ -168,12 +170,15 @@ export class DatasService {
{},
listArgs,
);
count = await baseModel.count(listArgs);
} catch (e) {
console.log(e);
NcError.internalServerError('Please check server log for more details');
NcError.internalServerError(
'Please check server log for more details',
);
}
return data;
})(),
]);
return new PagedResponseImpl(data, {
...query,
count,

4
tests/playwright/tests/db/features/filters.spec.ts

@ -856,14 +856,14 @@ test.describe('Filter Tests: AddOn', () => {
op: 'is checked',
value: null,
rowCount: records.list.filter(r => {
return r[dataType] === (context.dbType === 'pg' ? true : 1);
return r[dataType];
}).length,
},
{
op: 'is not checked',
value: null,
rowCount: records.list.filter(r => {
return r[dataType] !== (context.dbType === 'pg' ? true : 1);
return r[dataType];
}).length,
},
];

4
tests/playwright/tests/utils/general.ts

@ -37,8 +37,10 @@ function isSubset(obj, potentialSubset) {
if (!isSubset(objValue, potentialValue)) {
return false;
}
// skip strict type check since different database returns number values in string/number type
// todo: revert back to strict type check once we are consistent with type
// eslint-disable-next-line no-prototype-builtins
} else if (!obj.hasOwnProperty(prop) || objValue !== potentialValue) {
} else if (!obj.hasOwnProperty(prop) || objValue != potentialValue) {
return false;
}
}

Loading…
Cancel
Save