Browse Source

Merge branch 'develop' into feat/ajv-validation-followup

pull/5222/head
Wing-Kam Wong 2 years ago
parent
commit
de9a5f8328
  1. 84
      packages/nc-gui/components/smartsheet/toolbar/ColumnFilter.vue
  2. 8
      packages/nc-gui/components/smartsheet/toolbar/FilterInput.vue
  3. 20
      packages/nc-gui/composables/useViewFilters.ts
  4. 1
      packages/nc-gui/lang/en.json
  5. 230
      packages/nc-gui/utils/filterUtils.ts
  6. 31
      packages/noco-docs/content/en/developer-resources/rest-apis.md
  7. 1
      packages/nocodb-sdk/src/lib/Api.ts
  8. 2
      packages/nocodb/src/lib/Noco.ts
  9. 48
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts
  10. 160
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/conditionV2.ts
  11. 2
      packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts
  12. 45
      packages/nocodb/src/lib/meta/api/sync/helpers/job.ts
  13. 4
      packages/nocodb/src/lib/migrations/XcMigrationSourcev2.ts
  14. 16
      packages/nocodb/src/lib/migrations/v2/nc_027_add_comparison_sub_op.ts
  15. 91
      packages/nocodb/src/lib/models/Filter.ts
  16. 2
      packages/nocodb/src/lib/version-upgrader/NcUpgrader.ts
  17. 93
      packages/nocodb/src/lib/version-upgrader/ncFilterUpgrader_0105003.ts
  18. 5
      packages/nocodb/src/schema/swagger.json
  19. 6
      packages/nocodb/tests/unit/factory/row.ts
  20. 377
      packages/nocodb/tests/unit/rest/tests/filter.test.ts
  21. 2
      tests/playwright/pages/Dashboard/Grid/index.ts
  22. 70
      tests/playwright/pages/Dashboard/common/Toolbar/Filter.ts
  23. 6
      tests/playwright/setup/xcdb-records.ts
  24. 361
      tests/playwright/tests/filters.spec.ts

84
packages/nc-gui/components/smartsheet/toolbar/ColumnFilter.vue

@ -6,6 +6,7 @@ import {
MetaInj,
ReloadViewDataHookInj,
comparisonOpList,
comparisonSubOpList,
computed,
inject,
ref,
@ -54,6 +55,7 @@ const {
sync,
saveOrUpdateDebounced,
isComparisonOpAllowed,
isComparisonSubOpAllowed,
} = useViewFilters(
activeView,
parentId,
@ -75,9 +77,10 @@ const filterPrevComparisonOp = ref<Record<string, string>>({})
const filterUpdateCondition = (filter: FilterType, i: number) => {
const col = getColumn(filter)
if (!col) return
if (
col.uidt === UITypes.SingleSelect &&
['anyof', 'nanyof'].includes(filterPrevComparisonOp.value[filter.id]) &&
['anyof', 'nanyof'].includes(filterPrevComparisonOp.value[filter.id!]) &&
['eq', 'neq'].includes(filter.comparison_op!)
) {
// anyof and nanyof can allow multiple selections,
@ -87,12 +90,30 @@ const filterUpdateCondition = (filter: FilterType, i: number) => {
// since `blank`, `empty`, `null` doesn't require value,
// hence remove the previous value
filter.value = ''
filter.comparison_sub_op = ''
} else if ([UITypes.Date, UITypes.DateTime].includes(col.uidt as UITypes)) {
// for date / datetime,
// the input type could be decimal or datepicker / datetime picker
// hence remove the previous value
filter.value = ''
if (
!comparisonSubOpList(filter.comparison_op!)
.map((op) => op.value)
.includes(filter.comparison_sub_op!)
) {
if (filter.comparison_op === 'isWithin') {
filter.comparison_sub_op = 'pastNumberOfDays'
} else {
filter.comparison_sub_op = 'exactDate'
}
}
}
saveOrUpdate(filter, i)
filterPrevComparisonOp.value[filter.id] = filter.comparison_op
$e('a:filter:update', {
logical: filter.logical_op,
comparison: filter.comparison_op,
comparison_sub_op: filter.comparison_sub_op,
})
}
@ -109,7 +130,7 @@ const types = computed(() => {
watch(
() => activeView.value?.id,
(n: string, o: string) => {
(n, o) => {
// if nested no need to reload since it will get reloaded from parent
if (!nested && n !== o && (hookId || !webHook)) loadFilters(hookId as string)
},
@ -137,14 +158,28 @@ const applyChanges = async (hookId?: string, _nested = false) => {
}
const selectFilterField = (filter: Filter, index: number) => {
const col = getColumn(filter)
if (!col) return
// when we change the field,
// the corresponding default filter operator needs to be changed as well
// since the existing one may not be supported for the new field
// e.g. `eq` operator is not supported in checkbox field
// hence, get the first option of the supported operators of the new field
filter.comparison_op = comparisonOpList(getColumn(filter)!.uidt as UITypes).filter((compOp) =>
filter.comparison_op = comparisonOpList(col.uidt as UITypes).filter((compOp) =>
isComparisonOpAllowed(filter, compOp),
)?.[0].value
if ([UITypes.Date, UITypes.DateTime].includes(col.uidt as UITypes) && !['blank', 'notblank'].includes(filter.comparison_op)) {
if (filter.comparison_op === 'isWithin') {
filter.comparison_sub_op = 'pastNumberOfDays'
} else {
filter.comparison_sub_op = 'exactDate'
}
} else {
// reset
filter.comparison_sub_op = ''
}
// reset filter value as well
filter.value = ''
saveOrUpdate(filter, index)
@ -261,24 +296,49 @@ defineExpose({
</template>
</a-select>
<span
<a-select
v-if="
filter.comparison_op &&
['null', 'notnull', 'checked', 'notchecked', 'empty', 'notempty', 'blank', 'notblank'].includes(
filter.comparison_op,
)
[UITypes.Date, UITypes.DateTime].includes(getColumn(filter)?.uidt) &&
!['blank', 'notblank'].includes(filter.comparison_op)
"
:key="`span${i}`"
/>
v-model:value="filter.comparison_sub_op"
:dropdown-match-select-width="false"
class="caption nc-filter-sub_operation-select"
:placeholder="$t('labels.operationSub')"
density="compact"
variant="solo"
:disabled="filter.readOnly"
hide-details
dropdown-class-name="nc-dropdown-filter-comp-sub-op"
@change="filterUpdateCondition(filter, i)"
>
<template v-for="compSubOp of comparisonSubOpList(filter.comparison_op)" :key="compSubOp.value">
<a-select-option v-if="isComparisonSubOpAllowed(filter, compSubOp)" :value="compSubOp.value">
{{ compSubOp.text }}
</a-select-option>
</template>
</a-select>
<span v-else />
<a-checkbox
v-else-if="filter.field && types[filter.field] === 'boolean'"
v-if="filter.field && types[filter.field] === 'boolean'"
v-model:checked="filter.value"
dense
:disabled="filter.readOnly"
@change="saveOrUpdate(filter, i)"
/>
<span
v-else-if="
filter.comparison_sub_op
? comparisonSubOpList(filter.comparison_op).find((op) => op.value === filter.comparison_sub_op)?.ignoreVal ??
false
: comparisonOpList(getColumn(filter)?.uidt).find((op) => op.value === filter.comparison_op)?.ignoreVal ?? false
"
:key="`span${i}`"
/>
<LazySmartsheetToolbarFilterInput
v-else
class="nc-filter-value-select min-w-[120px]"
@ -315,7 +375,7 @@ defineExpose({
<style scoped>
.nc-filter-grid {
grid-template-columns: auto auto auto auto auto;
grid-template-columns: auto auto auto auto auto auto;
@apply grid gap-[12px] items-center;
}

8
packages/nc-gui/components/smartsheet/toolbar/FilterInput.vue

@ -117,9 +117,13 @@ const componentMap: Partial<Record<FilterType, any>> = $computed(() => {
// use MultiSelect for SingleSelect columns for anyof / nanyof filters
isSingleSelect: ['anyof', 'nanyof'].includes(props.filter.comparison_op!) ? MultiSelect : SingleSelect,
isMultiSelect: MultiSelect,
isDate: DatePicker,
isDate: ['daysAgo', 'daysFromNow', 'pastNumberOfDays', 'nextNumberOfDays'].includes(props.filter.comparison_sub_op!)
? Decimal
: DatePicker,
isYear: YearPicker,
isDateTime: DateTimePicker,
isDateTime: ['daysAgo', 'daysFromNow', 'pastNumberOfDays', 'nextNumberOfDays'].includes(props.filter.comparison_sub_op!)
? Decimal
: DateTimePicker,
isTime: TimePicker,
isRating: Rating,
isDuration: Duration,

20
packages/nc-gui/composables/useViewFilters.ts

@ -140,6 +140,25 @@ export function useViewFilters(
return isNullOrEmptyOp ? projectMeta.value.showNullAndEmptyInFilter : true
}
const isComparisonSubOpAllowed = (
filter: FilterType,
compOp: {
text: string
value: string
ignoreVal?: boolean
includedTypes?: UITypes[]
excludedTypes?: UITypes[]
},
) => {
if (compOp.includedTypes) {
// include allowed values only if selected column type matches
return filter.fk_column_id && compOp.includedTypes.includes(types.value[filter.fk_column_id])
} else if (compOp.excludedTypes) {
// include not allowed values only if selected column type not matches
return filter.fk_column_id && !compOp.excludedTypes.includes(types.value[filter.fk_column_id])
}
}
const placeholderFilter = (): Filter => {
return {
comparison_op: comparisonOpList(options.value?.[0].uidt as UITypes).filter((compOp) =>
@ -327,5 +346,6 @@ export function useViewFilters(
addFilterGroup,
saveOrUpdateDebounced,
isComparisonOpAllowed,
isComparisonSubOpAllowed,
}
}

1
packages/nc-gui/lang/en.json

@ -235,6 +235,7 @@
"action": "Action",
"actions": "Actions",
"operation": "Operation",
"operationSub": "Sub Operation",
"operationType": "Operation type",
"operationSubType": "Operation sub-type",
"description": "Description",

230
packages/nc-gui/utils/filterUtils.ts

@ -3,7 +3,11 @@ import { UITypes, isNumericCol, numericUITypes } from 'nocodb-sdk'
const getEqText = (fieldUiType: UITypes) => {
if (isNumericCol(fieldUiType)) {
return '='
} else if ([UITypes.SingleSelect, UITypes.Collaborator, UITypes.LinkToAnotherRecord].includes(fieldUiType)) {
} else if (
[UITypes.SingleSelect, UITypes.Collaborator, UITypes.LinkToAnotherRecord, UITypes.Date, UITypes.DateTime].includes(
fieldUiType,
)
) {
return 'is'
}
return 'is equal'
@ -12,7 +16,11 @@ const getEqText = (fieldUiType: UITypes) => {
const getNeqText = (fieldUiType: UITypes) => {
if (isNumericCol(fieldUiType)) {
return '!='
} else if ([UITypes.SingleSelect, UITypes.Collaborator, UITypes.LinkToAnotherRecord].includes(fieldUiType)) {
} else if (
[UITypes.SingleSelect, UITypes.Collaborator, UITypes.LinkToAnotherRecord, UITypes.Date, UITypes.DateTime].includes(
fieldUiType,
)
) {
return 'is not'
}
return 'is not equal'
@ -32,12 +40,40 @@ const getNotLikeText = (fieldUiType: UITypes) => {
return 'is not like'
}
const getGtText = (fieldUiType: UITypes) => {
if ([UITypes.Date, UITypes.DateTime].includes(fieldUiType)) {
return 'is after'
}
return '>'
}
const getLtText = (fieldUiType: UITypes) => {
if ([UITypes.Date, UITypes.DateTime].includes(fieldUiType)) {
return 'is before'
}
return '<'
}
const getGteText = (fieldUiType: UITypes) => {
if ([UITypes.Date, UITypes.DateTime].includes(fieldUiType)) {
return 'is on or after'
}
return '>='
}
const getLteText = (fieldUiType: UITypes) => {
if ([UITypes.Date, UITypes.DateTime].includes(fieldUiType)) {
return 'is on or before'
}
return '<='
}
export const comparisonOpList = (
fieldUiType: UITypes,
): {
text: string
value: string
ignoreVal?: boolean
ignoreVal: boolean
includedTypes?: UITypes[]
excludedTypes?: UITypes[]
}[] => [
@ -56,22 +92,42 @@ export const comparisonOpList = (
{
text: getEqText(fieldUiType),
value: 'eq',
ignoreVal: false,
excludedTypes: [UITypes.Checkbox, UITypes.MultiSelect, UITypes.Attachment],
},
{
text: getNeqText(fieldUiType),
value: 'neq',
ignoreVal: false,
excludedTypes: [UITypes.Checkbox, UITypes.MultiSelect, UITypes.Attachment],
},
{
text: getLikeText(fieldUiType),
value: 'like',
excludedTypes: [UITypes.Checkbox, UITypes.SingleSelect, UITypes.MultiSelect, UITypes.Collaborator, ...numericUITypes],
ignoreVal: false,
excludedTypes: [
UITypes.Checkbox,
UITypes.SingleSelect,
UITypes.MultiSelect,
UITypes.Collaborator,
UITypes.Date,
UITypes.DateTime,
...numericUITypes,
],
},
{
text: getNotLikeText(fieldUiType),
value: 'nlike',
excludedTypes: [UITypes.Checkbox, UITypes.SingleSelect, UITypes.MultiSelect, UITypes.Collaborator, ...numericUITypes],
ignoreVal: false,
excludedTypes: [
UITypes.Checkbox,
UITypes.SingleSelect,
UITypes.MultiSelect,
UITypes.Collaborator,
UITypes.Date,
UITypes.DateTime,
...numericUITypes,
],
},
{
text: 'is empty',
@ -85,6 +141,8 @@ export const comparisonOpList = (
UITypes.Attachment,
UITypes.LinkToAnotherRecord,
UITypes.Lookup,
UITypes.Date,
UITypes.DateTime,
...numericUITypes,
],
},
@ -100,6 +158,8 @@ export const comparisonOpList = (
UITypes.Attachment,
UITypes.LinkToAnotherRecord,
UITypes.Lookup,
UITypes.Date,
UITypes.DateTime,
...numericUITypes,
],
},
@ -116,6 +176,8 @@ export const comparisonOpList = (
UITypes.Attachment,
UITypes.LinkToAnotherRecord,
UITypes.Lookup,
UITypes.Date,
UITypes.DateTime,
],
},
{
@ -131,47 +193,63 @@ export const comparisonOpList = (
UITypes.Attachment,
UITypes.LinkToAnotherRecord,
UITypes.Lookup,
UITypes.Date,
UITypes.DateTime,
],
},
{
text: 'contains all of',
value: 'allof',
ignoreVal: false,
includedTypes: [UITypes.MultiSelect],
},
{
text: 'contains any of',
value: 'anyof',
ignoreVal: false,
includedTypes: [UITypes.MultiSelect, UITypes.SingleSelect],
},
{
text: 'does not contain all of',
value: 'nallof',
ignoreVal: false,
includedTypes: [UITypes.MultiSelect],
},
{
text: 'does not contain any of',
value: 'nanyof',
ignoreVal: false,
includedTypes: [UITypes.MultiSelect, UITypes.SingleSelect],
},
{
text: '>',
text: getGtText(fieldUiType),
value: 'gt',
includedTypes: [...numericUITypes],
ignoreVal: false,
includedTypes: [...numericUITypes, UITypes.Date, UITypes.DateTime],
},
{
text: '<',
text: getLtText(fieldUiType),
value: 'lt',
includedTypes: [...numericUITypes],
ignoreVal: false,
includedTypes: [...numericUITypes, UITypes.Date, UITypes.DateTime],
},
{
text: '>=',
text: getGteText(fieldUiType),
value: 'gte',
includedTypes: [...numericUITypes],
ignoreVal: false,
includedTypes: [...numericUITypes, UITypes.Date, UITypes.DateTime],
},
{
text: '<=',
text: getLteText(fieldUiType),
value: 'lte',
includedTypes: [...numericUITypes],
ignoreVal: false,
includedTypes: [...numericUITypes, UITypes.Date, UITypes.DateTime],
},
{
text: 'is within',
value: 'isWithin',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'is blank',
@ -186,3 +264,129 @@ export const comparisonOpList = (
excludedTypes: [UITypes.Checkbox],
},
]
export const comparisonSubOpList = (
// TODO: type
comparison_op: string,
): {
text: string
value: string
ignoreVal: boolean
includedTypes?: UITypes[]
excludedTypes?: UITypes[]
}[] => {
if (comparison_op === 'isWithin') {
return [
{
text: 'the past week',
value: 'pastWeek',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the past month',
value: 'pastMonth',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the past year',
value: 'pastYear',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the next week',
value: 'nextWeek',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the next month',
value: 'nextMonth',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the next year',
value: 'nextYear',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the next number of days',
value: 'nextNumberOfDays',
ignoreVal: false,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'the past number of days',
value: 'pastNumberOfDays',
ignoreVal: false,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
]
}
return [
{
text: 'today',
value: 'today',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'tomorrow',
value: 'tomorrow',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'yesterday',
value: 'yesterday',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'one week ago',
value: 'oneWeekAgo',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'one week from now',
value: 'oneWeekFromNow',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'one month ago',
value: 'oneMonthAgo',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'one month from now',
value: 'oneMonthFromNow',
ignoreVal: true,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'number of days ago',
value: 'daysAgo',
ignoreVal: false,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'number of days from now',
value: 'daysFromNow',
ignoreVal: false,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
{
text: 'exact date',
value: 'exactDate',
ignoreVal: false,
includedTypes: [UITypes.Date, UITypes.DateTime],
},
]
}

31
packages/noco-docs/content/en/developer-resources/rest-apis.md

@ -221,11 +221,42 @@ Currently, the default value for {orgs} is <b>noco</b>. Users will be able to ch
| btw | between | (colName,btw,val1,val2) |
| nbtw | not between | (colName,nbtw,val1,val2) |
| like | like | (colName,like,%name) |
| isWithin | is Within (Available in `Date` and `DateTime` only) | (colName,isWithin,sub_op) |
| allof | includes all of | (colName,allof,val1,val2,...) |
| anyof | includes any of | (colName,anyof,val1,val2,...) |
| nallof | does not include all of (includes none or some, but not all of) | (colName,nallof,val1,val2,...) |
| nanyof | does not include any of (includes none of) | (colName,nanyof,val1,val2,...) |
## Comparison Sub-Operators
The following sub-operators are available in `Date` and `DateTime` columns.
| Operation | Meaning | Example |
|-----------------|-------------------------|-----------------------------------|
| today | today | (colName,eq,today) |
| tomorrow | tomorrow | (colName,eq,tomorrow) |
| yesterday | yesterday | (colName,eq,yesterday) |
| oneWeekAgo | one week ago | (colName,eq,oneWeekAgo) |
| oneWeekFromNow | one week from now | (colName,eq,oneWeekFromNow) |
| oneMonthAgo | one month ago | (colName,eq,oneMonthAgo) |
| oneMonthFromNow | one month from now | (colName,eq,oneMonthFromNow) |
| daysAgo | number of days ago | (colName,eq,daysAgo,10) |
| daysFromNow | number of days from now | (colName,eq,daysFromNow,10) |
| exactDate | exact date | (colName,eq,exactDate,2022-02-02) |
For `isWithin` in `Date` and `DateTime` columns, the different set of sub-operators are used.
| Operation | Meaning | Example |
|------------------|-------------------------|-----------------------------------------|
| pastWeek | the past week | (colName,isWithin,pastWeek) |
| pastMonth | the past month | (colName,isWithin,pastMonth) |
| pastYear | the past year | (colName,isWithin,pastYear) |
| nextWeek | the next week | (colName,isWithin,nextWeek) |
| nextMonth | the next month | (colName,isWithin,nextMonth) |
| nextYear | the next year | (colName,isWithin,nextYear) |
| nextNumberOfDays | the next number of days | (colName,isWithin,nextNumberOfDays,10) |
| pastNumberOfDays | the past number of days | (colName,isWithin,pastNumberOfDays,10) |
## Logical Operators
| Operation | Example |

1
packages/nocodb-sdk/src/lib/Api.ts

@ -388,6 +388,7 @@ export interface FilterType {
fk_column_id?: IdType;
logical_op?: string;
comparison_op?: string;
comparison_sub_op?: string;
value?: any;
is_group?: boolean | number | null;
children?: FilterType[];

2
packages/nocodb/src/lib/Noco.ts

@ -105,7 +105,7 @@ export default class Noco {
constructor() {
process.env.PORT = process.env.PORT || '8080';
// todo: move
process.env.NC_VERSION = '0105002';
process.env.NC_VERSION = '0105003';
// if env variable NC_MINIMAL_DBS is set, then disable project creation with external sources
if (process.env.NC_MINIMAL_DBS) {

48
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@ -11,7 +11,11 @@ import DataLoader from 'dataloader';
import Column from '../../../../models/Column';
import { XcFilter, XcFilterWithAlias } from '../BaseModel';
import conditionV2 from './conditionV2';
import Filter from '../../../../models/Filter';
import Filter, {
COMPARISON_OPS,
COMPARISON_SUB_OPS,
IS_WITHIN_COMPARISON_SUB_OPS,
} from '../../../../models/Filter';
import sortV2 from './sortV2';
import Sort from '../../../../models/Sort';
import FormulaColumn from '../../../../models/FormulaColumn';
@ -2934,6 +2938,7 @@ function extractFilterFromXwhere(
if (openIndex === -1) openIndex = str.indexOf('(~');
let nextOpenIndex = openIndex;
let closingIndex = str.indexOf('))');
// if it's a simple query simply return array of conditions
@ -2986,15 +2991,54 @@ function extractFilterFromXwhere(
return nestedArrayConditions;
}
// mark `op` and `sub_op` any for being assignable to parameter of type
function validateFilterComparison(uidt: UITypes, op: any, sub_op?: any) {
if (!COMPARISON_OPS.includes(op)) {
NcError.badRequest(`${op} is not supported.`);
}
if (sub_op) {
if (![UITypes.Date, UITypes.DateTime].includes(uidt)) {
NcError.badRequest(`'${sub_op}' is not supported for UI Type'${uidt}'.`);
}
if (!COMPARISON_SUB_OPS.includes(sub_op)) {
NcError.badRequest(`'${sub_op}' is not supported.`);
}
if (
(op === 'isWithin' && !IS_WITHIN_COMPARISON_SUB_OPS.includes(sub_op)) ||
(op !== 'isWithin' && IS_WITHIN_COMPARISON_SUB_OPS.includes(sub_op))
) {
NcError.badRequest(`'${sub_op}' is not supported for '${op}'`);
}
}
}
function extractCondition(nestedArrayConditions, aliasColObjMap) {
return nestedArrayConditions?.map((str) => {
// eslint-disable-next-line prefer-const
let [logicOp, alias, op, value] =
str.match(/(?:~(and|or|not))?\((.*?),(\w+),(.*)\)/)?.slice(1) || [];
if (op === 'in') value = value.split(',');
let sub_op = null;
if (aliasColObjMap[alias]) {
if (
[UITypes.Date, UITypes.DateTime].includes(aliasColObjMap[alias].uidt)
) {
value = value.split(',');
// the first element would be sub_op
sub_op = value[0];
// remove the first element which is sub_op
value.shift();
} else if (op === 'in') {
value = value.split(',');
}
validateFilterComparison(aliasColObjMap[alias].uidt, op, sub_op);
}
return new Filter({
comparison_op: op,
...(sub_op && { comparison_sub_op: sub_op }),
fk_column_id: aliasColObjMap[alias]?.id,
logical_op: logicOp,
value,

160
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/conditionV2.ts

@ -8,8 +8,12 @@ import genRollupSelectv2 from './genRollupSelectv2';
import RollupColumn from '../../../../models/RollupColumn';
import formulaQueryBuilderv2 from './formulav2/formulaQueryBuilderv2';
import FormulaColumn from '../../../../models/FormulaColumn';
import { RelationTypes, UITypes, isNumericCol } from 'nocodb-sdk';
import { isNumericCol, RelationTypes, UITypes } from 'nocodb-sdk';
import { sanitize } from './helpers/sanitize';
import dayjs, { extend } from 'dayjs';
import customParseFormat from 'dayjs/plugin/customParseFormat.js';
extend(customParseFormat);
export default async function conditionV2(
conditionObj: Filter | Filter[],
@ -271,14 +275,83 @@ const parseConditionV2 = async (
return (qb: Knex.QueryBuilder) => {
let [field, val] = [_field, _val];
if (
[UITypes.Date, UITypes.DateTime].includes(column.uidt) &&
!val &&
['is', 'isnot'].includes(filter.comparison_op)
) {
// for date & datetime,
// val cannot be empty for non-is & non-isnot filters
return;
const dateFormat =
qb?.client?.config?.client === 'mysql2'
? 'YYYY-MM-DD HH:mm:ss'
: 'YYYY-MM-DD HH:mm:ssZ';
if ([UITypes.Date, UITypes.DateTime].includes(column.uidt)) {
const now = dayjs(new Date());
// handle sub operation
switch (filter.comparison_sub_op) {
case 'today':
val = now;
break;
case 'tomorrow':
val = now.add(1, 'day');
break;
case 'yesterday':
val = now.add(-1, 'day');
break;
case 'oneWeekAgo':
val = now.add(-1, 'week');
break;
case 'oneWeekFromNow':
val = now.add(1, 'week');
break;
case 'oneMonthAgo':
val = now.add(-1, 'month');
break;
case 'oneMonthFromNow':
val = now.add(1, 'month');
break;
case 'daysAgo':
if (!val) return;
val = now.add(-val, 'day');
break;
case 'daysFromNow':
if (!val) return;
val = now.add(val, 'day');
break;
case 'exactDate':
if (!val) return;
break;
// sub-ops for `isWithin` comparison
case 'pastWeek':
val = now.add(-1, 'week');
break;
case 'pastMonth':
val = now.add(-1, 'month');
break;
case 'pastYear':
val = now.add(-1, 'year');
break;
case 'nextWeek':
val = now.add(1, 'week');
break;
case 'nextMonth':
val = now.add(1, 'month');
break;
case 'nextYear':
val = now.add(1, 'year');
break;
case 'pastNumberOfDays':
if (!val) return;
val = now.add(-val, 'day');
break;
case 'nextNumberOfDays':
if (!val) return;
val = now.add(val, 'day');
break;
}
if (dayjs.isDayjs(val)) {
// turn `val` in dayjs object format to string
val = val.format(dateFormat).toString();
// keep YYYY-MM-DD only for date
val = column.uidt === UITypes.Date ? val.substring(0, 10) : val;
}
}
if (isNumericCol(column.uidt) && typeof val === 'string') {
@ -481,6 +554,27 @@ const parseConditionV2 = async (
}
}
break;
case 'lt':
const lt_op = customWhereClause ? '>' : '<';
qb = qb.where(field, lt_op, val);
if (column.uidt === UITypes.Rating) {
// unset number is considered as NULL
if (lt_op === '<' && val > 0) {
qb = qb.orWhereNull(field);
}
}
break;
case 'le':
case 'lte':
const le_op = customWhereClause ? '>=' : '<=';
qb = qb.where(field, le_op, val);
if (column.uidt === UITypes.Rating) {
// unset number is considered as NULL
if (le_op === '<=' || (le_op === '>=' && val === 0)) {
qb = qb.orWhereNull(field);
}
}
break;
case 'in':
qb = qb.whereIn(
field,
@ -517,27 +611,6 @@ const parseConditionV2 = async (
else if (filter.value === 'false')
qb = qb.whereNot(customWhereClause || field, false);
break;
case 'lt':
const lt_op = customWhereClause ? '>' : '<';
qb = qb.where(field, lt_op, val);
if (column.uidt === UITypes.Rating) {
// unset number is considered as NULL
if (lt_op === '<' && val > 0) {
qb = qb.orWhereNull(field);
}
}
break;
case 'le':
case 'lte':
const le_op = customWhereClause ? '>=' : '<=';
qb = qb.where(field, le_op, val);
if (column.uidt === UITypes.Rating) {
// unset number is considered as NULL
if (le_op === '<=' || (le_op === '>=' && val === 0)) {
qb = qb.orWhereNull(field);
}
}
break;
case 'empty':
if (column.uidt === UITypes.Formula) {
[field, val] = [val, field];
@ -564,7 +637,10 @@ const parseConditionV2 = async (
.orWhere(field, 'null');
} else {
qb = qb.whereNull(customWhereClause || field);
if (!isNumericCol(column.uidt)) {
if (
!isNumericCol(column.uidt) &&
![UITypes.Date, UITypes.DateTime].includes(column.uidt)
) {
qb = qb.orWhere(field, '');
}
}
@ -577,7 +653,10 @@ const parseConditionV2 = async (
.whereNot(field, 'null');
} else {
qb = qb.whereNotNull(customWhereClause || field);
if (!isNumericCol(column.uidt)) {
if (
!isNumericCol(column.uidt) &&
![UITypes.Date, UITypes.DateTime].includes(column.uidt)
) {
qb = qb.whereNot(field, '');
}
}
@ -598,6 +677,23 @@ const parseConditionV2 = async (
case 'nbtw':
qb = qb.whereNotBetween(field, val.split(','));
break;
case 'isWithin':
let now = dayjs(new Date()).format(dateFormat).toString();
now = column.uidt === UITypes.Date ? now.substring(0, 10) : now;
switch (filter.comparison_sub_op) {
case 'pastWeek':
case 'pastMonth':
case 'pastYear':
case 'pastNumberOfDays':
qb = qb.whereBetween(field, [val, now]);
break;
case 'nextWeek':
case 'nextMonth':
case 'nextYear':
case 'nextNumberOfDays':
qb = qb.whereBetween(field, [now, val]);
break;
}
}
};
}

2
packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts

@ -753,7 +753,7 @@ async function _formulaQueryBuilder(
sql = `${sql} ${colAlias}`;
}
}
const query = knex.raw(sql);
const query = knex.raw(sql.replace(/\?/g, '\\?'));
if (prevBinaryOp && pt.operator !== prevBinaryOp) {
query.wrap('(', ')');
}

45
packages/nocodb/src/lib/meta/api/sync/helpers/job.ts

@ -11,7 +11,6 @@ import hash from 'object-hash';
import { promisify } from 'util';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
import tinycolor from 'tinycolor2';
import { importData, importLTARData } from './readAndProcessData';
@ -19,8 +18,6 @@ import EntityMap from './EntityMap';
const writeJsonFileAsync = promisify(jsonfile.writeFile);
dayjs.extend(utc);
const selectColors = {
// normal
blue: '#cfdfff',
@ -679,8 +676,6 @@ export default async (
);
}
// debug
// console.log(JSON.stringify(tables, null, 2));
return tables;
}
@ -914,8 +909,6 @@ export default async (
aTblLinkColumns[i].name + suffix,
ncTbl.id
);
// console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name))
}
}
}
@ -1413,7 +1406,7 @@ export default async (
case UITypes.DateTime:
case UITypes.CreateTime:
case UITypes.LastModifiedTime:
rec[key] = dayjs(value).utc().format('YYYY-MM-DD HH:mm');
rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm');
break;
case UITypes.Date:
@ -1422,7 +1415,7 @@ export default async (
rec[key] = null;
logBasic(`:: Invalid date ${value}`);
} else {
rec[key] = dayjs(value).utc().format('YYYY-MM-DD');
rec[key] = dayjs(value).format('YYYY-MM-DD');
}
break;
@ -1504,8 +1497,6 @@ export default async (
})
.eachPage(
async function page(records, fetchNextPage) {
// console.log(JSON.stringify(records, null, 2));
// This function (`page`) will get called for each page of records.
// records.forEach(record => callback(table, record));
logBasic(
@ -1974,6 +1965,7 @@ export default async (
'>=': 'gte',
isEmpty: 'empty',
isNotEmpty: 'notempty',
isWithin: 'isWithin',
contains: 'like',
doesNotContain: 'nlike',
isAnyOf: 'anyof',
@ -2002,16 +1994,29 @@ export default async (
const datatype = colSchema.uidt;
const ncFilters = [];
// console.log(filter)
if (datatype === UITypes.Date || datatype === UITypes.DateTime) {
// skip filters over data datatype
updateMigrationSkipLog(
await sMap.getNcNameFromAtId(viewId),
colSchema.title,
colSchema.uidt,
`filter config skipped; filter over date datatype not supported`
);
continue;
let comparison_op = null;
let comparison_sub_op = null;
let value = null;
if (['isEmpty', 'isNotEmpty'].includes(filter.operator)) {
comparison_op = filter.operator === 'isEmpty' ? 'blank' : 'notblank';
} else {
if ('numberOfDays' in filter.value) {
value = filter.value['numberOfDays'];
} else if ('exactDate' in filter.value) {
value = filter.value['exactDate'];
}
comparison_op = filterMap[filter.operator];
comparison_sub_op = filter.value.mode;
}
const fx = {
fk_column_id: columnId,
logical_op: f.conjunction,
comparison_op,
comparison_sub_op,
value,
};
ncFilters.push(fx);
}
// single-select & multi-select

4
packages/nocodb/src/lib/migrations/XcMigrationSourcev2.ts

@ -14,6 +14,7 @@ import * as nc_023_multiple_source from './v2/nc_023_multiple_source';
import * as nc_024_barcode_column_type from './v2/nc_024_barcode_column_type';
import * as nc_025_add_row_height from './v2/nc_025_add_row_height';
import * as nc_026_map_view from './v2/nc_026_map_view';
import * as nc_027_add_comparison_sub_op from './v2/nc_027_add_comparison_sub_op';
// Create a custom migration source class
export default class XcMigrationSourcev2 {
@ -39,6 +40,7 @@ export default class XcMigrationSourcev2 {
'nc_024_barcode_column_type',
'nc_025_add_row_height',
'nc_026_map_view',
'nc_027_add_comparison_sub_op',
]);
}
@ -80,6 +82,8 @@ export default class XcMigrationSourcev2 {
return nc_025_add_row_height;
case 'nc_026_map_view':
return nc_026_map_view;
case 'nc_027_add_comparison_sub_op':
return nc_027_add_comparison_sub_op;
}
}
}

16
packages/nocodb/src/lib/migrations/v2/nc_027_add_comparison_sub_op.ts

@ -0,0 +1,16 @@
import { Knex } from 'knex';
import { MetaTable } from '../../utils/globals';
const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.FILTER_EXP, (table) => {
table.string('comparison_sub_op');
});
};
const down = async (knex) => {
await knex.schema.alterTable(MetaTable.FILTER_EXP, (table) => {
table.dropColumns('comparison_sub_op');
});
};
export { up, down };

91
packages/nocodb/src/lib/models/Filter.ts

@ -14,6 +14,63 @@ import NocoCache from '../cache/NocoCache';
import { NcError } from '../meta/helpers/catchError';
import { extractProps } from '../meta/helpers/extractProps';
export const COMPARISON_OPS = <const>[
'eq',
'neq',
'not',
'like',
'nlike',
'empty',
'notempty',
'null',
'notnull',
'checked',
'notchecked',
'blank',
'notblank',
'allof',
'anyof',
'nallof',
'nanyof',
'gt',
'lt',
'gte',
'lte',
'ge',
'le',
'in',
'isnot',
'is',
'isWithin',
'btw',
'nbtw',
];
export const IS_WITHIN_COMPARISON_SUB_OPS = <const>[
'pastWeek',
'pastMonth',
'pastYear',
'nextWeek',
'nextMonth',
'nextYear',
'pastNumberOfDays',
'nextNumberOfDays',
];
export const COMPARISON_SUB_OPS = <const>[
'today',
'tomorrow',
'yesterday',
'oneWeekAgo',
'oneWeekFromNow',
'oneMonthAgo',
'oneMonthFromNow',
'daysAgo',
'daysFromNow',
'exactDate',
...IS_WITHIN_COMPARISON_SUB_OPS,
];
export default class Filter {
id: string;
@ -23,35 +80,9 @@ export default class Filter {
fk_column_id?: string;
fk_parent_id?: string;
comparison_op?:
| 'eq'
| 'neq'
| 'not'
| 'like'
| 'nlike'
| 'empty'
| 'notempty'
| 'null'
| 'notnull'
| 'checked'
| 'notchecked'
| 'blank'
| 'notblank'
| 'allof'
| 'anyof'
| 'nallof'
| 'nanyof'
| 'gt'
| 'lt'
| 'gte'
| 'lte'
| 'ge'
| 'le'
| 'in'
| 'isnot'
| 'is'
| 'btw'
| 'nbtw';
comparison_op?: typeof COMPARISON_OPS[number];
comparison_sub_op?: typeof COMPARISON_SUB_OPS[number];
value?: string;
logical_op?: string;
@ -86,6 +117,7 @@ export default class Filter {
'fk_hook_id',
'fk_column_id',
'comparison_op',
'comparison_sub_op',
'value',
'fk_parent_id',
'is_group',
@ -223,6 +255,7 @@ export default class Filter {
const updateObj = extractProps(filter, [
'fk_column_id',
'comparison_op',
'comparison_sub_op',
'value',
'fk_parent_id',
'is_group',

2
packages/nocodb/src/lib/version-upgrader/NcUpgrader.ts

@ -13,6 +13,7 @@ import ncAttachmentUpgrader from './ncAttachmentUpgrader';
import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002';
import ncStickyColumnUpgrader from './ncStickyColumnUpgrader';
import ncFilterUpgrader_0104004 from './ncFilterUpgrader_0104004';
import ncFilterUpgrader_0105003 from './ncFilterUpgrader_0105003';
const log = debug('nc:version-upgrader');
import boxen from 'boxen';
@ -45,6 +46,7 @@ export default class NcUpgrader {
{ name: '0104002', handler: ncAttachmentUpgrader_0104002 },
{ name: '0104004', handler: ncFilterUpgrader_0104004 },
{ name: '0105002', handler: ncStickyColumnUpgrader },
{ name: '0105003', handler: ncFilterUpgrader_0105003 },
];
if (!(await ctx.ncMeta.knexConnection?.schema?.hasTable?.('nc_store'))) {
return;

93
packages/nocodb/src/lib/version-upgrader/ncFilterUpgrader_0105003.ts

@ -0,0 +1,93 @@
import { NcUpgraderCtx } from './NcUpgrader';
import { MetaTable } from '../utils/globals';
import NcMetaIO from '../meta/NcMetaIO';
import Column from '../models/Column';
import Filter from '../models/Filter';
import { UITypes } from 'nocodb-sdk';
// as of 0.105.3, date / datetime filters include `is like` and `is not like` which are not practical
// `removeLikeAndNlikeFilters` in this upgrader is simply to remove them
// besides, `null` and `empty` will be migrated to `blank` in `migrateEmptyAndNullFilters`
// since the upcoming version will introduce a set of new filters for date / datetime with a new `comparison_sub_op`
// `eq` and `neq` would become `is` / `is not` (comparison_op) + `exact date` (comparison_sub_op)
// `migrateEqAndNeqFilters` in this upgrader is to add `exact date` in comparison_sub_op
// Change Summary:
// - Date / DateTime columns:
// - remove `is like` and `is not like`
// - migrate `null` or `empty` filters to `blank`
// - add `exact date` in comparison_sub_op for existing filters `eq` and `neq`
function removeLikeAndNlikeFilters(filter: Filter, ncMeta: NcMetaIO) {
let actions = [];
// remove `is like` and `is not like`
if (['like', 'nlike'].includes(filter.comparison_op)) {
actions.push(Filter.delete(filter.id, ncMeta));
}
return actions;
}
function migrateEqAndNeqFilters(filter: Filter, ncMeta: NcMetaIO) {
let actions = [];
// remove `is like` and `is not like`
if (['eq', 'neq'].includes(filter.comparison_op)) {
actions.push(
Filter.update(
filter.id,
{
comparison_sub_op: 'exactDate',
},
ncMeta
)
);
}
return actions;
}
function migrateEmptyAndNullFilters(filter: Filter, ncMeta: NcMetaIO) {
let actions = [];
// remove `is like` and `is not like`
if (['empty', 'null'].includes(filter.comparison_op)) {
// migrate to blank
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'blank',
},
ncMeta
)
);
} else if (['notempty', 'notnull'].includes(filter.comparison_op)) {
// migrate to not blank
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'notblank',
},
ncMeta
)
);
}
return actions;
}
export default async function ({ ncMeta }: NcUpgraderCtx) {
const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
for (const filter of filters) {
if (!filter.fk_column_id || filter.is_group) {
continue;
}
const col = await Column.get({ colId: filter.fk_column_id }, ncMeta);
if ([UITypes.Date, UITypes.DateTime].includes(col.uidt)) {
await Promise.all([
...removeLikeAndNlikeFilters(filter, ncMeta),
...migrateEmptyAndNullFilters(filter, ncMeta),
...migrateEqAndNeqFilters(filter, ncMeta),
]);
}
}
}

5
packages/nocodb/src/schema/swagger.json

@ -1723,6 +1723,7 @@
"fk_column_id": "string",
"logical_op": "string",
"comparison_op": "string",
"comparison_sub_op": "string",
"value": "string",
"is_group": true,
"children": [{}]
@ -7557,6 +7558,7 @@
"fk_column_id": "string",
"logical_op": "string",
"comparison_op": "string",
"comparison_sub_op": "string",
"value": "string",
"is_group": true,
"children": [{}]
@ -7740,6 +7742,9 @@
"comparison_op": {
"type": "string"
},
"comparison_sub_op": {
"type": "string"
},
"value": {},
"is_group": {
"oneOf": [

6
packages/nocodb/tests/unit/factory/row.ts

@ -173,7 +173,11 @@ const rowMixedValue = (column: ColumnType, index: number) => {
case UITypes.LongText:
return longText[index % longText.length];
case UITypes.Date:
return '2020-01-01';
// set startDate as 400 days before today
// eslint-disable-next-line no-case-declarations
const result = new Date();
result.setDate(result.getDate() - 400 + index);
return result.toISOString().slice(0, 10);
case UITypes.URL:
return urls[index % urls.length];
case UITypes.SingleSelect:

377
packages/nocodb/tests/unit/rest/tests/filter.test.ts

@ -578,8 +578,385 @@ function filterSelectBased() {
});
}
async function applyDateFilter(filterParams, expectedRecords) {
const response = await request(context.app)
.get(`/api/v1/db/data/noco/${project.id}/${table.id}`)
.set('xc-auth', context.token)
.query({
filterArrJson: JSON.stringify([filterParams]),
})
.expect(200);
// expect(response.body.pageInfo.totalRows).to.equal(expectedRecords);
if (response.body.pageInfo.totalRows !== expectedRecords) {
console.log('filterParams', filterParams);
console.log(
'response.body.pageInfo.totalRows',
response.body.pageInfo.totalRows
);
console.log('expectedRecords', expectedRecords);
}
return response.body.list;
}
function filterDateBased() {
// prepare data for test cases
beforeEach(async function () {
context = await init();
project = await createProject(context);
table = await createTable(context, project, {
table_name: 'dateBased',
title: 'dateBased',
columns: [
{
column_name: 'Id',
title: 'Id',
uidt: UITypes.ID,
},
{
column_name: 'Date',
title: 'Date',
uidt: UITypes.Date,
},
],
});
columns = await table.getColumns();
let rowAttributes = [];
for (let i = 0; i < 800; i++) {
let row = {
Date: rowMixedValue(columns[1], i),
};
rowAttributes.push(row);
}
await createBulkRows(context, {
project,
table,
values: rowAttributes,
});
unfilteredRecords = await listRow({ project, table });
// verify length of unfiltered records to be 800
expect(unfilteredRecords.length).to.equal(800);
});
it('Type: Date ', async () => {
const today = new Date().setHours(0, 0, 0, 0);
const tomorrow = new Date(
new Date().setDate(new Date().getDate() + 1)
).setHours(0, 0, 0, 0);
const yesterday = new Date(
new Date().setDate(new Date().getDate() - 1)
).setHours(0, 0, 0, 0);
const oneWeekAgo = new Date(
new Date().setDate(new Date().getDate() - 7)
).setHours(0, 0, 0, 0);
const oneWeekFromNow = new Date(
new Date().setDate(new Date().getDate() + 7)
).setHours(0, 0, 0, 0);
const oneMonthAgo = new Date(
new Date().setMonth(new Date().getMonth() - 1)
).setHours(0, 0, 0, 0);
const oneMonthFromNow = new Date(
new Date().setMonth(new Date().getMonth() + 1)
).setHours(0, 0, 0, 0);
const daysAgo45 = new Date(
new Date().setDate(new Date().getDate() - 45)
).setHours(0, 0, 0, 0);
const daysFromNow45 = new Date(
new Date().setDate(new Date().getDate() + 45)
).setHours(0, 0, 0, 0);
const thisMonth15 = new Date(new Date().setDate(15)).setHours(0, 0, 0, 0);
const oneYearAgo = new Date(
new Date().setFullYear(new Date().getFullYear() - 1)
).setHours(0, 0, 0, 0);
const oneYearFromNow = new Date(
new Date().setFullYear(new Date().getFullYear() + 1)
).setHours(0, 0, 0, 0);
// records array with time set to 00:00:00; store time in unix epoch
const recordsTimeSetToZero = unfilteredRecords.map((r) => {
const date = new Date(r['Date']);
date.setHours(0, 0, 0, 0);
return date.getTime();
});
const isFilterList = [
{
opSub: 'today',
rowCount: recordsTimeSetToZero.filter((r) => r === today).length,
},
{
opSub: 'tomorrow',
rowCount: recordsTimeSetToZero.filter((r) => r === tomorrow).length,
},
{
opSub: 'yesterday',
rowCount: recordsTimeSetToZero.filter((r) => r === yesterday).length,
},
{
opSub: 'oneWeekAgo',
rowCount: recordsTimeSetToZero.filter((r) => r === oneWeekAgo).length,
},
{
opSub: 'oneWeekFromNow',
rowCount: recordsTimeSetToZero.filter((r) => r === oneWeekFromNow)
.length,
},
{
opSub: 'oneMonthAgo',
rowCount: recordsTimeSetToZero.filter((r) => r === oneMonthAgo).length,
},
{
opSub: 'oneMonthFromNow',
rowCount: recordsTimeSetToZero.filter((r) => r === oneMonthFromNow)
.length,
},
{
opSub: 'daysAgo',
value: 45,
rowCount: recordsTimeSetToZero.filter((r) => r === daysAgo45).length,
},
{
opSub: 'daysFromNow',
value: 45,
rowCount: recordsTimeSetToZero.filter((r) => r === daysFromNow45)
.length,
},
{
opSub: 'exactDate',
value: new Date(thisMonth15).toISOString().split('T')[0],
rowCount: recordsTimeSetToZero.filter((r) => r === thisMonth15).length,
},
];
// "is after" filter list
const isAfterFilterList = [
{
opSub: 'today',
rowCount: recordsTimeSetToZero.filter((r) => r > today).length,
},
{
opSub: 'tomorrow',
rowCount: recordsTimeSetToZero.filter((r) => r > tomorrow).length,
},
{
opSub: 'yesterday',
rowCount: recordsTimeSetToZero.filter((r) => r > yesterday).length,
},
{
opSub: 'oneWeekAgo',
rowCount: recordsTimeSetToZero.filter((r) => r > oneWeekAgo).length,
},
{
opSub: 'oneWeekFromNow',
rowCount: recordsTimeSetToZero.filter((r) => r > oneWeekFromNow).length,
},
{
opSub: 'oneMonthAgo',
rowCount: recordsTimeSetToZero.filter((r) => r > oneMonthAgo).length,
},
{
opSub: 'oneMonthFromNow',
rowCount: recordsTimeSetToZero.filter((r) => r > oneMonthFromNow)
.length,
},
{
opSub: 'daysAgo',
value: 45,
rowCount: recordsTimeSetToZero.filter((r) => r > daysAgo45).length,
},
{
opSub: 'daysFromNow',
value: 45,
rowCount: recordsTimeSetToZero.filter((r) => r > daysFromNow45).length,
},
{
opSub: 'exactDate',
value: new Date().toISOString().split('T')[0],
rowCount: recordsTimeSetToZero.filter((r) => r > today).length,
},
];
// "is within" filter list
const isWithinFilterList = [
{
opSub: 'pastWeek',
rowCount: recordsTimeSetToZero.filter(
(r) => r >= oneWeekAgo && r <= today
).length,
},
{
opSub: 'pastMonth',
rowCount: recordsTimeSetToZero.filter(
(r) => r >= oneMonthAgo && r <= today
).length,
},
{
opSub: 'pastYear',
rowCount: recordsTimeSetToZero.filter(
(r) => r >= oneYearAgo && r <= today
).length,
},
{
opSub: 'nextWeek',
rowCount: recordsTimeSetToZero.filter(
(r) => r >= today && r <= oneWeekFromNow
).length,
},
{
opSub: 'nextMonth',
rowCount: recordsTimeSetToZero.filter(
(r) => r >= today && r <= oneMonthFromNow
).length,
},
{
opSub: 'nextYear',
rowCount: recordsTimeSetToZero.filter(
(r) => r >= today && r <= oneYearFromNow
).length,
},
{
opSub: 'nextNumberOfDays',
value: 45,
rowCount: recordsTimeSetToZero.filter(
(r) => r >= today && r <= daysFromNow45
).length,
},
{
opSub: 'pastNumberOfDays',
value: 45,
rowCount: recordsTimeSetToZero.filter(
(r) => r >= daysAgo45 && r <= today
).length,
},
];
// rest of the filters (without subop type)
const filterList = [
{
opType: 'blank',
rowCount: unfilteredRecords.filter(
(r) => r['Date'] === null || r['Date'] === ''
).length,
},
{
opType: 'notblank',
rowCount: unfilteredRecords.filter(
(r) => r['Date'] !== null && r['Date'] !== ''
).length,
},
];
// is
for (let i = 0; i < isFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'eq',
comparison_sub_op: isFilterList[i].opSub,
value: isFilterList[i].value,
};
await applyDateFilter(filter, isFilterList[i].rowCount);
}
// is not
for (let i = 0; i < isFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'neq',
comparison_sub_op: isFilterList[i].opSub,
value: isFilterList[i].value,
};
await applyDateFilter(filter, 800 - isFilterList[i].rowCount);
}
// is before
for (let i = 0; i < isAfterFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'gt',
comparison_sub_op: isAfterFilterList[i].opSub,
value: isAfterFilterList[i].value,
};
await applyDateFilter(filter, isAfterFilterList[i].rowCount);
}
// is before or on
for (let i = 0; i < isAfterFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'gte',
comparison_sub_op: isAfterFilterList[i].opSub,
value: isAfterFilterList[i].value,
};
await applyDateFilter(filter, isAfterFilterList[i].rowCount + 1);
}
// is after
for (let i = 0; i < isAfterFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'lt',
comparison_sub_op: isAfterFilterList[i].opSub,
value: isAfterFilterList[i].value,
};
await applyDateFilter(filter, 800 - isAfterFilterList[i].rowCount - 1);
}
// is after or on
for (let i = 0; i < isAfterFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'lte',
comparison_sub_op: isAfterFilterList[i].opSub,
value: isAfterFilterList[i].value,
};
await applyDateFilter(filter, 800 - isAfterFilterList[i].rowCount);
}
// is within
for (let i = 0; i < isWithinFilterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: 'isWithin',
comparison_sub_op: isWithinFilterList[i].opSub,
value: isWithinFilterList[i].value,
};
await applyDateFilter(filter, isWithinFilterList[i].rowCount);
}
// rest of the filters (without subop type)
for (let i = 0; i < filterList.length; i++) {
const filter = {
fk_column_id: columns[1].id,
status: 'create',
logical_op: 'and',
comparison_op: filterList[i].opType,
value: '',
};
await applyDateFilter(filter, filterList[i].rowCount);
}
});
}
export default function () {
describe('Filter: Text based', filterTextBased);
describe('Filter: Numerical', filterNumberBased);
describe('Filter: Select based', filterSelectBased);
describe('Filter: Date based', filterDateBased);
}

2
tests/playwright/pages/Dashboard/Grid/index.ts

@ -212,8 +212,8 @@ export class GridPage extends BasePage {
recordCnt = records[0].split(' ')[0];
// to ensure page loading is complete
await this.rootPage.waitForTimeout(500);
i++;
await this.rootPage.waitForTimeout(100 * i);
}
expect(parseInt(recordCnt)).toEqual(count);
}

70
tests/playwright/pages/Dashboard/common/Toolbar/Filter.ts

@ -29,20 +29,28 @@ export class ToolbarFilterPage extends BasePage {
).toBeChecked();
}
async clickAddFilter() {
await this.get().locator(`button:has-text("Add Filter")`).first().click();
}
async add({
columnTitle,
opType,
opSubType,
value,
isLocallySaved,
dataType,
openModal = false,
}: {
columnTitle: string;
opType: string;
opSubType?: string; // for date datatype
value?: string;
isLocallySaved: boolean;
dataType?: string;
openModal?: boolean;
}) {
await this.get().locator(`button:has-text("Add Filter")`).first().click();
if (!openModal) await this.get().locator(`button:has-text("Add Filter")`).first().click();
const selectedField = await this.rootPage.locator('.nc-filter-field-select').textContent();
if (selectedField !== columnTitle) {
@ -53,19 +61,6 @@ export class ToolbarFilterPage extends BasePage {
.click();
}
// network request will be triggered only after filter value is configured
//
// const selectColumn = this.rootPage
// .locator('div.ant-select-dropdown.nc-dropdown-toolbar-field-list')
// .locator(`div[label="${columnTitle}"]`)
// .click();
// await this.waitForResponse({
// uiAction: selectColumn,
// httpMethodsToMatch: isLocallySaved ? ['GET'] : ['POST', 'PATCH'],
// requestUrlPathToMatch: isLocallySaved ? `/api/v1/db/public/` : `/filters`,
// });
// await this.toolbar.parent.dashboard.waitForLoaderToDisappear();
const selectedOpType = await this.rootPage.locator('.nc-filter-operation-select').textContent();
if (selectedOpType !== opType) {
await this.rootPage.locator('.nc-filter-operation-select').click();
@ -76,27 +71,42 @@ export class ToolbarFilterPage extends BasePage {
.first()
.click();
}
// if (selectedOpType !== opType) {
// await this.rootPage.locator('.nc-filter-operation-select').last().click();
// // first() : filter list has >, >=
// const selectOpType = this.rootPage
// .locator('.nc-dropdown-filter-comp-op')
// .locator(`.ant-select-item:has-text("${opType}")`)
// .first()
// .click();
//
// await this.waitForResponse({
// uiAction: selectOpType,
// httpMethodsToMatch: isLocallySaved ? ['GET'] : ['POST', 'PATCH'],
// requestUrlPathToMatch: isLocallySaved ? `/api/v1/db/public/` : `/filters`,
// });
// await this.toolbar.parent.dashboard.waitForLoaderToDisappear();
// }
// subtype for date
if (dataType === UITypes.Date && opSubType) {
const selectedSubType = await this.rootPage.locator('.nc-filter-sub_operation-select').textContent();
if (selectedSubType !== opSubType) {
await this.rootPage.locator('.nc-filter-sub_operation-select').click();
// first() : filter list has >, >=
await this.rootPage
.locator('.nc-dropdown-filter-comp-sub-op')
.locator(`.ant-select-item:has-text("${opSubType}")`)
.first()
.click();
}
}
// if value field was provided, fill it
if (value) {
let fillFilter: any = null;
switch (dataType) {
case UITypes.Date:
if (opSubType === 'exact date') {
await this.get().locator('.nc-filter-value-select').click();
await this.rootPage.locator(`.ant-picker-dropdown:visible`);
await this.rootPage.locator(`.ant-picker-cell-inner:has-text("${value}")`).click();
} else {
fillFilter = () => this.rootPage.locator('.nc-filter-value-select > input').last().fill(value);
await this.waitForResponse({
uiAction: fillFilter,
httpMethodsToMatch: ['GET'],
requestUrlPathToMatch: isLocallySaved ? `/api/v1/db/public/` : `/api/v1/db/data/noco/`,
});
await this.toolbar.parent.dashboard.waitForLoaderToDisappear();
await this.toolbar.parent.waitLoading();
break;
}
break;
case UITypes.Duration:
await this.get().locator('.nc-filter-value-select').locator('input').fill(value);
break;

6
tests/playwright/setup/xcdb-records.ts

@ -120,7 +120,11 @@ const rowMixedValue = (column: ColumnType, index: number) => {
case UITypes.LongText:
return longText[index % longText.length];
case UITypes.Date:
return '2020-01-01';
// set startDate as 400 days before today
// eslint-disable-next-line no-case-declarations
const result = new Date();
result.setDate(result.getDate() - 400 + index);
return result.toISOString().slice(0, 10);
case UITypes.URL:
return urls[index % urls.length];
case UITypes.SingleSelect:

361
tests/playwright/tests/filters.spec.ts

@ -44,9 +44,39 @@ async function validateRowArray(param) {
await dashboard.grid.verifyTotalRowCount({ count: rowCount });
}
async function verifyFilter_withFixedModal(param: {
column: string;
opType: string;
opSubType?: string;
value?: string;
result: { rowCount: number };
dataType?: string;
}) {
// if opType was included in skip list, skip it
if (skipList[param.column]?.includes(param.opType)) {
return;
}
await toolbar.filter.add({
columnTitle: param.column,
opType: param.opType,
opSubType: param.opSubType,
value: param.value,
isLocallySaved: false,
dataType: param?.dataType,
openModal: true,
});
// verify filtered rows
await validateRowArray({
rowCount: param.result.rowCount,
});
}
async function verifyFilter(param: {
column: string;
opType: string;
opSubType?: string;
value?: string;
result: { rowCount: number };
dataType?: string;
@ -56,10 +86,13 @@ async function verifyFilter(param: {
return;
}
console.log(`Verifying filter: ${param.opType} ${param.opSubType}`);
await toolbar.clickFilter();
await toolbar.filter.add({
columnTitle: param.column,
opType: param.opType,
opSubType: param.opSubType,
value: param.value,
isLocallySaved: false,
dataType: param?.dataType,
@ -591,6 +624,334 @@ test.describe('Filter Tests: Select based', () => {
});
});
// Date & Time related
//
test.describe('Filter Tests: Date based', () => {
const today = new Date().setHours(0, 0, 0, 0);
const tomorrow = new Date(new Date().setDate(new Date().getDate() + 1)).setHours(0, 0, 0, 0);
const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)).setHours(0, 0, 0, 0);
const oneWeekAgo = new Date(new Date().setDate(new Date().getDate() - 7)).setHours(0, 0, 0, 0);
const oneWeekFromNow = new Date(new Date().setDate(new Date().getDate() + 7)).setHours(0, 0, 0, 0);
const oneMonthAgo = new Date(new Date().setMonth(new Date().getMonth() - 1)).setHours(0, 0, 0, 0);
const oneMonthFromNow = new Date(new Date().setMonth(new Date().getMonth() + 1)).setHours(0, 0, 0, 0);
const daysAgo45 = new Date(new Date().setDate(new Date().getDate() - 45)).setHours(0, 0, 0, 0);
const daysFromNow45 = new Date(new Date().setDate(new Date().getDate() + 45)).setHours(0, 0, 0, 0);
const thisMonth15 = new Date(new Date().setDate(15)).setHours(0, 0, 0, 0);
const oneYearAgo = new Date(new Date().setFullYear(new Date().getFullYear() - 1)).setHours(0, 0, 0, 0);
const oneYearFromNow = new Date(new Date().setFullYear(new Date().getFullYear() + 1)).setHours(0, 0, 0, 0);
async function dateTimeBasedFilterTest(dataType) {
await dashboard.closeTab({ title: 'Team & Auth' });
await dashboard.treeView.openTable({ title: 'dateTimeBased' });
// Enable NULL & EMPTY filters
await dashboard.gotoSettings();
await dashboard.settings.toggleNullEmptyFilters();
// records array with time set to 00:00:00; store time in unix epoch
const recordsTimeSetToZero = records.list.map(r => {
const date = new Date(r[dataType]);
date.setHours(0, 0, 0, 0);
return date.getTime();
});
const isFilterList = [
{
opSub: 'today',
rowCount: recordsTimeSetToZero.filter(r => r === today).length,
},
{
opSub: 'tomorrow',
rowCount: recordsTimeSetToZero.filter(r => r === tomorrow).length,
},
{
opSub: 'yesterday',
rowCount: recordsTimeSetToZero.filter(r => r === yesterday).length,
},
{
opSub: 'one week ago',
rowCount: recordsTimeSetToZero.filter(r => r === oneWeekAgo).length,
},
{
opSub: 'one week from now',
rowCount: recordsTimeSetToZero.filter(r => r === oneWeekFromNow).length,
},
{
opSub: 'one month ago',
rowCount: recordsTimeSetToZero.filter(r => r === oneMonthAgo).length,
},
{
opSub: 'one month from now',
rowCount: recordsTimeSetToZero.filter(r => r === oneMonthFromNow).length,
},
{
opSub: 'number of days ago',
value: 45,
rowCount: recordsTimeSetToZero.filter(r => r === daysAgo45).length,
},
{
opSub: 'number of days from now',
value: 45,
rowCount: recordsTimeSetToZero.filter(r => r === daysFromNow45).length,
},
{
opSub: 'exact date',
value: 15,
rowCount: recordsTimeSetToZero.filter(r => r === thisMonth15).length,
},
];
// "is after" filter list
const isAfterFilterList = [
{
opSub: 'today',
rowCount: recordsTimeSetToZero.filter(r => r > today).length,
},
{
opSub: 'tomorrow',
rowCount: recordsTimeSetToZero.filter(r => r > tomorrow).length,
},
{
opSub: 'yesterday',
rowCount: recordsTimeSetToZero.filter(r => r > yesterday).length,
},
{
opSub: 'one week ago',
rowCount: recordsTimeSetToZero.filter(r => r > oneWeekAgo).length,
},
{
opSub: 'one week from now',
rowCount: recordsTimeSetToZero.filter(r => r > oneWeekFromNow).length,
},
{
opSub: 'one month ago',
rowCount: recordsTimeSetToZero.filter(r => r > oneMonthAgo).length,
},
{
opSub: 'one month from now',
rowCount: recordsTimeSetToZero.filter(r => r > oneMonthFromNow).length,
},
{
opSub: 'number of days ago',
value: 45,
rowCount: recordsTimeSetToZero.filter(r => r > daysAgo45).length,
},
{
opSub: 'number of days from now',
value: 45,
rowCount: recordsTimeSetToZero.filter(r => r > daysFromNow45).length,
},
{
opSub: 'exact date',
value: 15,
rowCount: recordsTimeSetToZero.filter(r => r > thisMonth15).length,
},
];
// "is within" filter list
const isWithinFilterList = [
{
opSub: 'the past week',
rowCount: recordsTimeSetToZero.filter(r => r >= oneWeekAgo && r <= today).length,
},
{
opSub: 'the past month',
rowCount: recordsTimeSetToZero.filter(r => r >= oneMonthAgo && r <= today).length,
},
{
opSub: 'the past year',
rowCount: recordsTimeSetToZero.filter(r => r >= oneYearAgo && r <= today).length,
},
{
opSub: 'the next week',
rowCount: recordsTimeSetToZero.filter(r => r >= today && r <= oneWeekFromNow).length,
},
{
opSub: 'the next month',
rowCount: recordsTimeSetToZero.filter(r => r >= today && r <= oneMonthFromNow).length,
},
{
opSub: 'the next year',
rowCount: recordsTimeSetToZero.filter(r => r >= today && r <= oneYearFromNow).length,
},
{
opSub: 'the next number of days',
value: 45,
rowCount: recordsTimeSetToZero.filter(r => r >= today && r <= daysFromNow45).length,
},
{
opSub: 'the past number of days',
value: 45,
rowCount: recordsTimeSetToZero.filter(r => r >= daysAgo45 && r <= today).length,
},
];
// rest of the filters (without subop type)
const filterList = [
{
opType: 'is blank',
rowCount: records.list.filter(r => r[dataType] === null || r[dataType] === '').length,
},
{
opType: 'is not blank',
rowCount: records.list.filter(r => r[dataType] !== null && r[dataType] !== '').length,
},
];
await toolbar.clickFilter();
await toolbar.filter.clickAddFilter();
// "is" filter list
for (let i = 0; i < isFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is',
opSubType: isFilterList[i].opSub,
value: isFilterList[i]?.value?.toString() || '',
result: { rowCount: isFilterList[i].rowCount },
dataType: dataType,
});
}
// mutually exclusive of "is" filter list
for (let i = 0; i < isFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is not',
opSubType: isFilterList[i].opSub,
value: isFilterList[i]?.value?.toString() || '',
result: { rowCount: 800 - isFilterList[i].rowCount },
dataType: dataType,
});
}
// "is before" filter list
for (let i = 0; i < isAfterFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is before',
opSubType: isAfterFilterList[i].opSub,
value: isAfterFilterList[i]?.value?.toString() || '',
result: { rowCount: 800 - isAfterFilterList[i].rowCount - 1 },
dataType: dataType,
});
}
// "is on or before" filter list
for (let i = 0; i < isAfterFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is on or before',
opSubType: isAfterFilterList[i].opSub,
value: isAfterFilterList[i]?.value?.toString() || '',
result: { rowCount: 800 - isAfterFilterList[i].rowCount },
dataType: dataType,
});
}
// "is after" filter list
for (let i = 0; i < isAfterFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is after',
opSubType: isAfterFilterList[i].opSub,
value: isAfterFilterList[i]?.value?.toString() || '',
result: { rowCount: isAfterFilterList[i].rowCount },
dataType: dataType,
});
}
// "is on or after" filter list
for (let i = 0; i < isAfterFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is on or after',
opSubType: isAfterFilterList[i].opSub,
value: isAfterFilterList[i]?.value?.toString() || '',
result: { rowCount: 1 + isAfterFilterList[i].rowCount },
dataType: dataType,
});
}
// "is within" filter list
for (let i = 0; i < isWithinFilterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: 'is within',
opSubType: isWithinFilterList[i].opSub,
value: isWithinFilterList[i]?.value?.toString() || '',
result: { rowCount: isWithinFilterList[i].rowCount },
dataType: dataType,
});
}
// "is blank" and "is not blank" filter list
for (let i = 0; i < filterList.length; i++) {
await verifyFilter_withFixedModal({
column: dataType,
opType: filterList[i].opType,
opSubType: null,
value: null,
result: { rowCount: filterList[i].rowCount },
dataType: dataType,
});
}
}
test.beforeEach(async ({ page }) => {
context = await setup({ page });
dashboard = new DashboardPage(page, context.project);
toolbar = dashboard.grid.toolbar;
api = new Api({
baseURL: `http://localhost:8080/`,
headers: {
'xc-auth': context.token,
},
});
const columns = [
{
column_name: 'Id',
title: 'Id',
uidt: UITypes.ID,
},
{
column_name: 'Date',
title: 'Date',
uidt: UITypes.Date,
},
];
try {
const project = await api.project.read(context.project.id);
const table = await api.base.tableCreate(context.project.id, project.bases?.[0].id, {
table_name: 'dateTimeBased',
title: 'dateTimeBased',
columns: columns,
});
const rowAttributes = [];
for (let i = 0; i < 800; i++) {
const row = {
Date: rowMixedValue(columns[1], i),
};
rowAttributes.push(row);
}
await api.dbTableRow.bulkCreate('noco', context.project.id, table.id, rowAttributes);
records = await api.dbTableRow.list('noco', context.project.id, table.id, { limit: 800 });
} catch (e) {
console.error(e);
}
});
test('Date : filters', async () => {
await dateTimeBasedFilterTest('Date');
});
});
// Misc : Checkbox
//

Loading…
Cancel
Save