Browse Source

Nc feat/type conversions rebased (#8680)

* feat: created inferTypes.ts

* chore: switch to sdk types

* feat: convert string to given type

* feat: convert string to rating

* fix: handle the case with multiple .

* feat: numeric decimal type conversion in postgres

* feat: add cast for non text fields

* refactor: move type casts to separate file

* feat: add casts for date, date-time and time

* doc: added function docs

* feat: added cast for year and rating

* feat: added cast for duration

* fix: cast for multi-select

* fix: cast for multi-select

* fix: cast for year

* feat: date conversion on best effort basis

* fix: single line text to select

* fix: any field to select

* lint: simplified expressions

* fix: user conversion

* fix: user conversion

* fix: date time conversion

* test: added test cases for type casts

* fix: SLT to User field

* fix: SLT to Long text, single select and multiselect

* chore: handle True/False & TRUE/FALSE in checkbox

* lint: fixed eslint issues

* chore: remove system fields as destination type when converting a field

* feat: show warning when changing column type

* fix: toned down edit modal

* test: click on update button during warning popup

* test: update selector

* test: fix type change flag

* fix: handle date format

* chore: auto focus update button

* fix: parameterize columnName and other values

* chore: removed number of digits limit for hour

* test: fix add-edit modal label

* fix: fixed missing column reference

* fix: handle missing date format

* fix: handle missing date format

* test: fix save routine mux

* test: fix barCode & QRCode save

* refactor: combined uiType filters

* fix: sanitise column name

* refactor: switch to some instead of find

* feat: created inferTypes.ts

* chore: switch to sdk types

* feat: convert string to given type

* feat: numeric decimal type conversion in postgres

* feat: add cast for non text fields

* refactor: move type casts to separate file

* feat: add casts for date, date-time and time

* doc: added function docs

* feat: added cast for year and rating

* feat: added cast for duration

* fix: cast for multi-select

* fix: cast for multi-select

* fix: cast for year

* feat: date conversion on best effort basis

* fix: single line text to select

* fix: user conversion

* fix: date time conversion

* fix: SLT to User field

* fix: SLT to Long text, single select and multiselect

* chore: handle True/False & TRUE/FALSE in checkbox

* lint: fixed eslint issues

* feat: show warning when changing column type

* fix: toned down edit modal

* test: click on update button during warning popup

* fix: handle date format

* chore: auto focus update button

* fix: parameterize columnName and other values

* chore: removed number of digits limit for hour

* fix: handle missing date format

* fix: handle missing date format

* test: fix save routine mux

* fix: revert removing verify

* fix: sanitise column name

* fix: pass context

* tests: remove duplicate statement

* fix: add context bypass for list method

* fix: disable type conversion for Formula, BarCode, QrCode

* fix: render confirm modal sing useDialog to avoid accidental closing

* refactor: construct context using column while getting colOptions data

---------

Co-authored-by: rohittp <tprohit9@gmail.com>
Co-authored-by: Pranav C <pranavxc@gmail.com>
pull/8597/merge
Raju Udava 3 weeks ago committed by GitHub
parent
commit
58e70afaac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 46
      packages/nc-gui/components/dlg/ColumnUpdateConfirm.vue
  2. 54
      packages/nc-gui/components/smartsheet/column/EditOrAdd.vue
  3. 72
      packages/nc-gui/helpers/typeConvertion.ts
  4. 2
      packages/nocodb/src/controllers/columns.controller.spec.ts
  5. 43
      packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts
  6. 79
      packages/nocodb/src/db/sql-client/lib/pg/constants.ts
  7. 12
      packages/nocodb/src/db/sql-client/lib/pg/pg.queries.ts
  8. 227
      packages/nocodb/src/db/sql-client/lib/pg/typeCast.ts
  9. 8
      packages/nocodb/src/models/Column.ts
  10. 166
      packages/nocodb/src/services/columns.service.ts
  11. 18
      packages/nocodb/src/utils/cloud/populateCloudPlugins.ts
  12. 2
      packages/nocodb/tests/unit/rest/index.test.ts
  13. 133
      packages/nocodb/tests/unit/rest/tests/typeCasts.test.ts
  14. 34
      tests/playwright/pages/Dashboard/Grid/Column/index.ts
  15. 4
      tests/playwright/tests/db/columns/columnUserSelect.spec.ts
  16. 2
      tests/playwright/tests/db/general/tableColumnOperation.spec.ts
  17. 2
      tests/playwright/tests/db/views/viewCalendar.spec.ts
  18. 2
      tests/playwright/tests/db/views/viewKanban.spec.ts

46
packages/nc-gui/components/dlg/ColumnUpdateConfirm.vue

@ -0,0 +1,46 @@
<script setup lang="ts">
const props = defineProps<{
visible?: boolean
saving?: boolean
}>()
const emit = defineEmits(['submit', 'cancel', 'update:visible'])
const visible = useVModel(props, 'visible', emit)
</script>
<template>
<GeneralModal v-model:visible="visible" size="small">
<div class="flex flex-col p-6" @click.stop>
<div class="flex flex-row pb-2 mb-4 font-medium text-lg border-b-1 border-gray-50 text-gray-800">Field Type Change</div>
<div class="mb-3 text-gray-800">
<div class="flex item-center gap-2 mb-4">
<component :is="iconMap.warning" id="nc-selected-item-icon" class="text-yellow-500 w-10 h-10" />
This action cannot be undone. Converting data types may result in data loss. Proceed with caution!
</div>
</div>
<slot name="entity-preview"></slot>
<div class="flex flex-row gap-x-2 mt-2.5 pt-2.5 justify-end">
<NcButton type="secondary" @click="visible = false">
{{ $t('general.cancel') }}
</NcButton>
<NcButton
key="submit"
autofocus
type="primary"
html-type="submit"
:loading="saving"
data-testid="nc-delete-modal-delete-btn"
@click="emit('submit')"
>
Update
<template #loading> Saving... </template>
</NcButton>
</div>
</div>
</GeneralModal>
</template>

54
packages/nc-gui/components/smartsheet/column/EditOrAdd.vue

@ -2,6 +2,7 @@
import type { ColumnReqType, ColumnType } from 'nocodb-sdk'
import { UITypes, UITypesName, isLinksOrLTAR, isSelfReferencingTableColumn, isSystemColumn, isVirtualCol } from 'nocodb-sdk'
import Icon from '../header/Icon.vue'
import MdiPlusIcon from '~icons/mdi/plus-circle-outline'
import MdiMinusIcon from '~icons/mdi/minus-circle-outline'
import MdiIdentifierIcon from '~icons/mdi/identifier'
@ -25,7 +26,6 @@ const emit = defineEmits(['submit', 'cancel', 'mounted', 'add', 'update'])
const {
formState,
column,
generateNewColumnMeta,
addOrUpdate,
onAlter,
@ -33,6 +33,7 @@ const {
validateInfos,
isEdit,
disableSubmitBtn,
column,
} = useColumnCreateStoreOrThrow()
const { getMeta } = useMetas()
@ -95,6 +96,9 @@ const onlyNameUpdateOnEditColumns = [
UITypes.LastModifiedTime,
UITypes.CreatedBy,
UITypes.LastModifiedBy,
UITypes.Formula,
UITypes.QrCode,
UITypes.Barcode,
]
// To close column type dropdown on escape and
@ -109,16 +113,20 @@ const geoDataToggleCondition = (t: { name: UITypes }) => {
const showDeprecated = ref(false)
const isSystemField = (t: { name: UITypes }) =>
[UITypes.CreatedBy, UITypes.CreatedTime, UITypes.LastModifiedBy, UITypes.LastModifiedTime].includes(t.name)
const uiFilters = (t: { name: UITypes; virtual?: number }) => {
const systemFiledNotEdited = !isSystemField(t) || formState.value.uidt === t.name || !isEdit.value
const geoDataToggle = geoDataToggleCondition(t) && (!isEdit.value || !t.virtual || t.name === formState.value.uidt)
const specificDBType = t.name === UITypes.SpecificDBType && isXcdbBase(meta.value?.source_id)
return systemFiledNotEdited && geoDataToggle && !specificDBType
}
const uiTypesOptions = computed<typeof uiTypes>(() => {
return [
...uiTypes
.filter(
(t) =>
geoDataToggleCondition(t) &&
(!isEdit.value || !t.virtual || t.name === formState.value.uidt) &&
(!t.deprecated || showDeprecated.value),
)
.filter((t) => !(t.name === UITypes.SpecificDBType && isXcdbBase(meta.value?.source_id))),
...uiTypes.filter(uiFilters),
...(!isEdit.value && meta?.value?.columns?.every((c) => !c.pk)
? [
{
@ -146,7 +154,9 @@ const reloadMetaAndData = async () => {
const saving = ref(false)
async function onSubmit() {
const warningVisible = ref(false)
const saveSubmitted = async () => {
if (readOnly.value) return
saving.value = true
@ -166,6 +176,25 @@ async function onSubmit() {
}
}
async function onSubmit() {
if (readOnly.value) return
// Show warning message if user tries to change type of column
if (isEdit.value && formState.value.uidt !== column.value?.uidt) {
warningVisible.value = true
const { close } = useDialog(resolveComponent('DlgColumnUpdateConfirm'), {
'visible': warningVisible,
'onUpdate:visible': (value) => (warningVisible.value = value),
'saving': saving,
'onSubmit': async () => {
close()
await saveSubmitted()
},
})
} else await saveSubmitted()
}
// focus and select the column name field
const antInput = ref()
watchEffect(() => {
@ -294,6 +323,7 @@ const filterOption = (input: string, option: { value: UITypes }) => {
<template>
<div
v-if="!warningVisible"
class="overflow-auto max-h-[max(80vh,500px)]"
:class="{
'bg-white': !props.fromTableExplorer,
@ -352,7 +382,7 @@ const filterOption = (input: string, option: { value: UITypes }) => {
v-model:value="formState.uidt"
show-search
class="nc-column-type-input !rounded-lg"
:disabled="isKanban || readOnly || (isEdit && !!onlyNameUpdateOnEditColumns.find((col) => col === column?.uidt))"
:disabled="isKanban || readOnly || (isEdit && !!onlyNameUpdateOnEditColumns.includes(column?.uidt))"
dropdown-class-name="nc-dropdown-column-type border-1 !rounded-lg border-gray-200"
:filter-option="filterOption"
@dropdown-visible-change="onDropdownChange"
@ -556,6 +586,7 @@ const filterOption = (input: string, option: { value: UITypes }) => {
@apply font-normal;
}
}
.nc-column-name-input,
:deep(.nc-formula-input),
:deep(.ant-form-item-control-input-content > input.ant-input) {
@ -617,6 +648,7 @@ const filterOption = (input: string, option: { value: UITypes }) => {
@apply border-gray-300;
box-shadow: 0px 0px 4px 0px rgba(0, 0, 0, 0.24);
}
&.ant-select-disabled .ant-select-selector {
box-shadow: none;
}

72
packages/nc-gui/helpers/typeConvertion.ts

@ -0,0 +1,72 @@
import { UITypes } from 'nocodb-sdk'
import { getCheckboxValue } from './parsers/parserHelpers'
/*
* @param {string} str - string with numbers
* @returns {number} - number extracted from string
*
* @example abc123 -> 123
* @example 12.3abc -> 12.3
* @example 12.3.2 -> 12.32
*/
function extractNumbers(str: string): number {
const parts = str.replace(/[^\d.]/g, '').split('.')
if (parts.length > 1) parts[0] += '.'
return parseFloat(parts.join(''))
}
/*
* @param {string} value - string value of duration
* @returns {number} - duration in seconds
*
* @example 1:30:00 -> 5400
* @example 1:30 -> 5400
* @example 90 -> 90
*/
function toDuration(value: string) {
if (value.includes(':')) {
const [hours, minutes, seconds] = value.split(':').map((v) => parseInt(v) || 0)
return hours * 3600 + minutes * 60 + seconds
}
return Math.floor(extractNumbers(value))
}
/*
* @param {string} value - string value to convert
* @param {string} type - type of the field to convert to
* @returns {number|string|boolean|array} - converted value
*
* @example convert('12.3', 'Number') -> 12
* @example convert('1a23', 'SingleLineText') -> '1a23'
* @example convert('1', 'Checkbox') -> true
*/
export function convert(value: string, type: string, limit = 100): unknown {
switch (type) {
case UITypes.SingleLineText:
case UITypes.SingleSelect:
case UITypes.LongText:
case UITypes.Email:
case UITypes.URL:
return value
case UITypes.Number:
return Math.floor(extractNumbers(value))
case UITypes.Decimal:
case UITypes.Currency:
return extractNumbers(value)
case UITypes.Percent:
case UITypes.Rating:
return Math.min(limit, Math.max(0, extractNumbers(value)))
case UITypes.Checkbox:
return getCheckboxValue(value)
case UITypes.Date:
case UITypes.DateTime:
case UITypes.Time:
return new Date(value)
case UITypes.Duration:
return toDuration(value)
case UITypes.MultiSelect:
return value.split(',').map((v) => v.trim())
}
}

2
packages/nocodb/src/controllers/columns.controller.spec.ts

@ -1,7 +1,7 @@
import { Test } from '@nestjs/testing';
import { ColumnsService } from '../services/columns.service';
import { ColumnsController } from './columns.controller';
import type { TestingModule } from '@nestjs/testing';
import { ColumnsService } from '~/services/columns.service';
describe('ColumnsController', () => {
let controller: ColumnsController;

43
packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts

@ -3,10 +3,16 @@ import knex from 'knex';
import isEmpty from 'lodash/isEmpty';
import mapKeys from 'lodash/mapKeys';
import find from 'lodash/find';
import { UITypes } from 'nocodb-sdk';
import KnexClient from '~/db/sql-client/lib/KnexClient';
import Debug from '~/db/util/Debug';
import Result from '~/db/util/Result';
import queries from '~/db/sql-client/lib/pg/pg.queries';
import {
formatColumn,
generateCastQuery,
} from '~/db/sql-client/lib/pg/typeCast';
import pgQueries from '~/db/sql-client/lib/pg/pg.queries';
const log = new Debug('PGClient');
@ -2883,13 +2889,45 @@ class PGClient extends KnexClient {
}
if (n.dt !== o.dt) {
query += this.genQuery(
`\nALTER TABLE ?? ALTER COLUMN ?? DROP DEFAULT;\n`,
[t, n.cn],
shouldSanitize,
);
if (
[
UITypes.Date,
UITypes.DateTime,
UITypes.Time,
UITypes.Duration,
].includes(n.uidt)
) {
query += pgQueries.dateConversionFunction.default.sql;
}
query += this.genQuery(
`\nALTER TABLE ?? ALTER COLUMN ?? TYPE ${this.sanitiseDataType(
n.dt,
)} USING ??::${this.sanitiseDataType(n.dt)};\n`,
[t, n.cn, n.cn],
)} USING `,
[t, n.cn],
shouldSanitize,
);
const castedColumn = formatColumn(
this.genQuery('??', [n.cn], shouldSanitize),
o.uidt,
);
const limit = typeof n.dtxp === 'number' ? n.dtxp : null;
const castQuery = generateCastQuery(
n.uidt,
n.dt,
castedColumn,
limit,
n.meta.date_format || 'YYYY-MM-DD',
);
query += this.genQuery(castQuery, [], shouldSanitize);
}
if (n.rqd !== o.rqd) {
@ -3027,4 +3065,5 @@ class PGClient extends KnexClient {
return result;
}
}
export default PGClient;

79
packages/nocodb/src/db/sql-client/lib/pg/constants.ts

@ -0,0 +1,79 @@
const MONTHS = [
'January',
'February',
'March',
'April',
'May',
'June',
'July',
'August',
'September',
'October',
'November',
'December',
];
const MONTHS_SHORT = MONTHS.map((m) => m.slice(0, 3));
const MONTHS_SHORT_LOWER = MONTHS_SHORT.map((m) => m.toLowerCase());
const MONTHS_SHORT_UPPER = MONTHS_SHORT.map((m) => m.toUpperCase());
const MONTHS_LOWER = MONTHS.map((m) => m.toLowerCase());
const MONTHS_UPPER = MONTHS.map((m) => m.toUpperCase());
const MONTH_FORMATS = {
MM: '[0-9]{1,2}',
Mon: '(' + MONTHS_SHORT.join('|') + ')',
MON: '(' + MONTHS_SHORT_UPPER.join('|') + ')',
mon: '(' + MONTHS_SHORT_LOWER.join('|') + ')',
Month: '(' + MONTHS.join('|') + ')',
MONTH: '(' + MONTHS_UPPER.join('|') + ')',
month: '(' + MONTHS_LOWER.join('|') + ')',
};
/*
* Map of date formats to their respective regex patterns.
*/
export const DATE_FORMATS = {
ymd: Object.keys(MONTH_FORMATS).map((format) => [
`Y-${format}-DD`,
`^\\d{1,4}[:\\- /]+${MONTH_FORMATS[format]}[:\\- /]+\\d{1,2}$`,
]),
dmy: Object.keys(MONTH_FORMATS).map((format) => [
`DD-${format}-Y`,
`^\\d{1,2}[:\\- /]+${MONTH_FORMATS[format]}[:\\- /]+\\d{1,4}$`,
]),
mdy: Object.keys(MONTH_FORMATS).map((format) => [
`${format}-DD-Y`,
`^${MONTH_FORMATS[format]}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,4}$`,
]),
empty: [['', '^.*$']],
};
/*
* Map of date time formats to their respective regex patterns.
*/
export const TIME_FORMATS = [
[
'HH24:MI:SS:MS:US',
'^\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,3}[:\\- /]+\\d*$',
],
[
'HH24:MI:SS:MS',
'^\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,3}$',
],
['HH24:MI:SS', '^\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,2}$'],
['HH24:MI', '^\\d{1,2}[:\\- /]+\\d{1,2}$'],
['HH24', '^\\d{1,2}$'],
[
'HH12:MI:SS:MS:US (AM|PM)',
'^\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,3}[:\\- /]+\\d* (AM|PM)$',
],
[
'HH12:MI:SS:MS (AM|PM)',
'^\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,3} (AM|PM)$',
],
[
'HH12:MI:SS (AM|PM)',
'^\\d{1,2}[:\\- /]+\\d{1,2}[:\\- /]+\\d{1,2} (AM|PM)$',
],
['HH12:MI (AM|PM)', '^\\d{1,2}[:\\- /]+\\d{1,2} (AM|PM)$'],
['HH12 (AM|PM)', '^\\d{1,2} (AM|PM)$'],
];

12
packages/nocodb/src/db/sql-client/lib/pg/pg.queries.ts

@ -234,6 +234,18 @@ AND t.table_name=?;`,
paramsHints: ['databaseName'],
},
},
dateConversionFunction: {
default: {
sql: `CREATE OR REPLACE FUNCTION to_date_time_safe(value text, format text) RETURNS TIMESTAMP AS $$
BEGIN
RETURN to_timestamp(value, format);
EXCEPTION
WHEN others THEN RETURN NULL;
END;
$$ LANGUAGE plpgsql;`,
paramsHints: [],
},
},
};
export default pgQueries;

227
packages/nocodb/src/db/sql-client/lib/pg/typeCast.ts

@ -0,0 +1,227 @@
import { UITypes } from 'nocodb-sdk';
import { DATE_FORMATS, TIME_FORMATS } from '~/db/sql-client/lib/pg/constants';
/*
* Generate query to extract number from a string. The number is extracted by
* removing all non-numeric characters from the string. Decimal point is allowed.
* If there are more than one decimal points, only the first one is considered, the rest are ignored.
*
* @param {String} source - source column name
* @returns {String} - query to extract number from a string
*/
function extractNumberQuery(source: string) {
return `
CAST(
NULLIF(
REPLACE(
REPLACE(
REGEXP_REPLACE(
REGEXP_REPLACE(${source}, '[^0-9.]', '', 'g'),
'(\\d)\\.', '\\1-'
),
'.', ''
),
'-', '.'
), ''
) AS DECIMAL
)
`;
}
/*
* Generate query to cast a value to boolean. The boolean value is determined based on the given mappings.
*
* @param {String} columnName - Source column name
* @returns {String} - query to cast value to boolean
*/
function generateBooleanCastQuery(columnName: string): string {
return `
CASE
WHEN LOWER(${columnName}) IN ('checked', 'x', 'yes', 'y', '1', '[x]', '☑', '✅', '✓', '✔', 'enabled', 'on', 'done', 'true') THEN true
WHEN LOWER(${columnName}) IN ('unchecked', '', 'no', 'n', '0', '[]', '[ ]', 'disabled', 'off', 'false') THEN false
ELSE null
END;
`;
}
/*
* Generate query to cast a value to date time based on the given date and time formats.
*
* @param {String} source - Source column name
* @param {String} dateFormat - Date format
* @param {String} timeFormat - Time format
* @param {String} functionName - Function name to cast value to date time
* @returns {String} - query to cast value to date time
*/
function generateDateTimeCastQuery(source: string, dateFormat: string) {
if (!(dateFormat in DATE_FORMATS)) {
throw new Error(`Invalid date format: ${dateFormat}`);
}
const timeFormats =
dateFormat === 'empty' ? TIME_FORMATS : [...TIME_FORMATS, ['', '^$']];
const cases = DATE_FORMATS[dateFormat].map(([format, regex]) =>
timeFormats
.map(
([timeFormat, timeRegex]) =>
`WHEN ${source} ~ '${regex.slice(0, -1)}\\s*${timeRegex.slice(
1,
)}' THEN to_date_time_safe(${source}, '${format} ${timeFormat}')`,
)
.join('\n'),
);
return `CASE
${cases.join('\n')}
ELSE NULL
END;`;
}
/*
* Generate SQL query to extract a number from a string and make out-of-bounds values NULL.
*
* @param {String} source - Source column name.
* @param {Number} minValue - Minimum allowed value.
* @param {Number} maxValue - Maximum allowed value.
* @returns {String} - SQL query to extract number and handle out-of-bounds values.
*/
function generateNumberBoundingQuery(
source: string,
minValue: number,
maxValue: number,
) {
return `
NULLIF(
NULLIF(
LEAST(
${maxValue + 1}, GREATEST(${minValue - 1}, ${source})
), ${minValue - 1}
), ${maxValue + 1}
);
`;
}
/*
* Generate query to cast a value to duration.
*
* @param {String} source - Source column name
* @returns {String} - query to cast value to duration
*/
function generateToDurationQuery(source: string) {
return `
CASE
WHEN ${source} ~ '^\\d+:\\d{1,2}$' THEN 60 * CAST(SPLIT_PART(${source}, ':', 1) AS INT) + CAST(SPLIT_PART(${source}, ':', 2) AS INT)
ELSE ${extractNumberQuery(source)}
END;
`;
}
function getDateFormat(format: string) {
const y = format.indexOf('Y');
const m = format.indexOf('M');
const d = format.indexOf('D');
if (y < m) {
if (m < d) return 'ymd';
else if (y < d) return 'ydm';
else return 'dym';
} else if (y < d) return 'myd';
else if (m < d) return 'mdy';
else return 'dmy';
}
/*
* Generate query to cast a column to a specific data type based on the UI data type.
*
* @param {UITypes} uidt - UI data type
* @param {String} dt - DB Data type
* @param {String} source - Source column name
* @param {Number} limit - Limit for the data type
* @param {String} dateFormat - Date format
* @param {String} timeFormat - Time format
* @returns {String} - query to cast column to a specific data type
*/
export function generateCastQuery(
uidt: UITypes,
dt: string,
source: string,
limit: number,
format: string,
) {
switch (uidt) {
case UITypes.SingleLineText:
case UITypes.MultiSelect:
case UITypes.SingleSelect:
case UITypes.Email:
case UITypes.PhoneNumber:
case UITypes.URL:
return `${source}::VARCHAR(${limit || 255});`;
case UITypes.LongText:
return `${source}::TEXT;`;
case UITypes.Number:
return `CAST(${extractNumberQuery(source)} AS BIGINT);`;
case UITypes.Year:
return generateNumberBoundingQuery(
extractNumberQuery(source),
1000,
9999,
);
case UITypes.Decimal:
case UITypes.Currency:
return `${extractNumberQuery(source)};`;
case UITypes.Percent:
return `LEAST(100, GREATEST(0, ${extractNumberQuery(source)}));`;
case UITypes.Rating:
return `LEAST(${limit || 5}, GREATEST(0, ${extractNumberQuery(
source,
)}));`;
case UITypes.Checkbox:
return generateBooleanCastQuery(source);
case UITypes.Date:
return `CAST(${generateDateTimeCastQuery(
source,
getDateFormat(format),
).slice(0, -1)} AS DATE);`;
case UITypes.DateTime:
return generateDateTimeCastQuery(source, getDateFormat(format));
case UITypes.Time:
return generateDateTimeCastQuery(source, 'empty');
case UITypes.Duration:
return generateToDurationQuery(source);
default:
return `null::${dt};`;
}
}
/*
* Generate query to format a column based on the UI data type.
*
* @param {String} columnName - Column name
* @param {UITypes} uiDataType - UI data type
* @returns {String} - query to format a column
*/
export function formatColumn(columnName: string, uiDataType: UITypes) {
switch (uiDataType) {
case UITypes.LongText:
case UITypes.SingleLineText:
case UITypes.MultiSelect:
case UITypes.Email:
case UITypes.URL:
case UITypes.SingleSelect:
case UITypes.PhoneNumber:
return columnName;
case UITypes.Number:
case UITypes.Decimal:
case UITypes.Currency:
case UITypes.Percent:
case UITypes.Rating:
case UITypes.Duration:
case UITypes.Year:
return `CAST(${columnName} AS VARCHAR(255))`;
case UITypes.Checkbox:
return `CAST(CASE WHEN ${columnName} THEN '1' ELSE '0' END AS TEXT)`;
default:
return `CAST(${columnName} AS TEXT)`;
}
}

8
packages/nocodb/src/models/Column.ts

@ -651,7 +651,13 @@ export default class Column<T = any> implements ColumnType {
}
if (colData) {
const column = new Column(colData);
await column.getColOptions(context, ncMeta);
await column.getColOptions(
{
workspace_id: column.fk_workspace_id,
base_id: column.base_id,
},
ncMeta,
);
return column;
}
return null;

166
packages/nocodb/src/services/columns.service.ts

@ -516,14 +516,7 @@ export class ColumnsService {
],
);
}
} else if (
[
UITypes.SingleLineText,
UITypes.Email,
UITypes.PhoneNumber,
UITypes.URL,
].includes(column.uidt)
) {
} else {
// Text to SingleSelect/MultiSelect
const dbDriver = await reuseOrSave('dbDriver', reuse, async () =>
NcConnectionMgrv2.get(source),
@ -555,7 +548,7 @@ export class ColumnsService {
);
const options = data.reduce((acc, el) => {
if (el[column.column_name]) {
const values = el[column.column_name].split(',');
const values = String(el[column.column_name]).split(',');
if (values.length > 1) {
if (colBody.uidt === UITypes.SingleSelect) {
NcError.badRequest(
@ -706,10 +699,11 @@ export class ColumnsService {
// Handle option delete
if (column.colOptions?.options) {
for (const option of column.colOptions.options.filter((oldOp) =>
colBody.colOptions.options.find((newOp) => newOp.id === oldOp.id)
? false
: true,
for (const option of column.colOptions.options.filter(
(oldOp) =>
!colBody.colOptions.options.find(
(newOp) => newOp.id === oldOp.id,
),
)) {
if (
!supportedDrivers.includes(driverType) &&
@ -1124,7 +1118,9 @@ export class ColumnsService {
});
} else if (colBody.uidt === UITypes.User) {
// handle default value for user column
if (colBody.cdf) {
if (typeof colBody.cdf !== 'string') {
colBody.cdf = '';
} else if (colBody.cdf) {
const baseUsers = await BaseUser.getUsersList(context, {
base_id: source.base_id,
include_ws_deleted: false,
@ -1257,9 +1253,7 @@ export class ColumnsService {
await Column.update(context, param.columnId, {
...colBody,
});
} else if (
[UITypes.SingleLineText, UITypes.Email].includes(column.uidt)
) {
} else {
// email/text to user
const baseModel = await reuseOrSave('baseModel', reuse, async () =>
Model.getBaseModelSQL(context, {
@ -1274,58 +1268,52 @@ export class ColumnsService {
base_id: column.base_id,
});
try {
const data = await baseModel.execAndParse(
sqlClient.knex
.raw('SELECT DISTINCT ?? FROM ??', [
column.column_name,
baseModel.getTnPath(table.table_name),
])
.toQuery(),
);
const data = await baseModel.execAndParse(
sqlClient.knex
.raw('SELECT DISTINCT ?? FROM ??', [
column.column_name,
baseModel.getTnPath(table.table_name),
])
.toQuery(),
);
let isMultiple = false;
const rows = data.map((el) => el[column.column_name]);
const rows = data.map((el) => el[column.column_name]);
const emails = rows
.map((el) => {
const res = el.split(',').map((e) => e.trim());
if (res.length > 1) {
isMultiple = true;
}
return res;
})
.flat();
if (rows.some((el) => el?.split(',').length > 1)) {
colBody.meta = {
is_multi: true,
};
}
// check if emails are present baseUsers
const emailsNotPresent = emails.filter((el) => {
return !baseUsers.find((user) => user.email === el);
});
// create nested replace statement for each user
let setStatement = 'null';
if (emailsNotPresent.length) {
NcError.badRequest(
`Some of the emails are not present in the database.`,
);
}
if (
[
UITypes.URL,
UITypes.Email,
UITypes.SingleLineText,
UITypes.PhoneNumber,
UITypes.SingleLineText,
UITypes.LongText,
UITypes.MultiSelect,
].includes(column.uidt)
) {
setStatement = baseUsers
.map((user) =>
sqlClient.knex
.raw('WHEN ?? = ? THEN ?', [
column.column_name,
user.email,
user.id,
])
.toQuery(),
)
.join('\n');
if (isMultiple) {
colBody.meta = {
is_multi: true,
};
}
} catch (e) {
NcError.badRequest('Some of the emails are present in the database.');
setStatement = `CASE\n${setStatement}\nELSE null\nEND`;
}
// create nested replace statement for each user
const setStatement = baseUsers.reduce((acc, user) => {
const qb = sqlClient.knex.raw(`REPLACE(${acc}, ?, ?)`, [
user.email,
user.id,
]);
return qb.toQuery();
}, sqlClient.knex.raw(`??`, [column.column_name]).toQuery());
await sqlClient.raw(`UPDATE ?? SET ?? = ${setStatement};`, [
baseModel.getTnPath(table.table_name),
column.column_name,
@ -1373,12 +1361,9 @@ export class ColumnsService {
await Column.update(context, param.columnId, {
...colBody,
});
} else {
NcError.notImplemented(`Updating ${column.uidt} => ${colBody.uidt}`);
}
} else if (column.uidt === UITypes.User) {
if ([UITypes.SingleLineText, UITypes.Email].includes(colBody.uidt)) {
// user to email/text
} else {
if (column.uidt === UITypes.User) {
const baseModel = await reuseOrSave('baseModel', reuse, async () =>
Model.getBaseModelSQL(context, {
id: table.id,
@ -1405,53 +1390,8 @@ export class ColumnsService {
baseModel.getTnPath(table.table_name),
column.column_name,
]);
colBody = await getColumnPropsFromUIDT(colBody, source);
const tableUpdateBody = {
...table,
tn: table.table_name,
originalColumns: table.columns.map((c) => ({
...c,
cn: c.column_name,
cno: c.column_name,
})),
columns: await Promise.all(
table.columns.map(async (c) => {
if (c.id === param.columnId) {
const res = {
...c,
...colBody,
cn: colBody.column_name,
cno: c.column_name,
altered: Altered.UPDATE_COLUMN,
};
// update formula with new column name
await this.updateFormulas(context, {
oldColumn: column,
colBody,
});
return Promise.resolve(res);
} else {
(c as any).cn = c.column_name;
}
return Promise.resolve(c);
}),
),
};
const sqlMgr = await reuseOrSave('sqlMgr', reuse, async () =>
ProjectMgrv2.getSqlMgr(context, { id: source.base_id }),
);
await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody);
await Column.update(context, param.columnId, {
...colBody,
});
} else {
NcError.notImplemented(`Updating ${column.uidt} => ${colBody.uidt}`);
}
} else {
colBody = await getColumnPropsFromUIDT(colBody, source);
const tableUpdateBody = {
...table,

18
packages/nocodb/src/utils/cloud/populateCloudPlugins.ts

@ -47,15 +47,15 @@ export const populatePluginsForCloud = async ({ ncMeta = Noco.ncMeta }) => {
);
}
// SES
if (
!process.env.NC_CLOUD_SES_ACCESS_KEY ||
!process.env.NC_CLOUD_SES_ACCESS_SECRET ||
!process.env.NC_CLOUD_SES_REGION ||
!process.env.NC_CLOUD_SES_FROM
) {
throw new Error('SES env variables not found');
}
// // SES
// if (
// !process.env.NC_CLOUD_SES_ACCESS_KEY ||
// !process.env.NC_CLOUD_SES_ACCESS_SECRET ||
// !process.env.NC_CLOUD_SES_REGION ||
// !process.env.NC_CLOUD_SES_FROM
// ) {
// throw new Error('SES env variables not found');
// }
const sesPluginData = await ncMeta.metaGet2(null, null, MetaTable.PLUGIN, {
title: SESPluginConfig.title,

2
packages/nocodb/tests/unit/rest/index.test.ts

@ -11,6 +11,7 @@ import filterTest from './tests/filter.test';
import newDataApisTest from './tests/newDataApis.test';
import groupByTest from './tests/groupby.test';
import formulaTests from './tests/formula.test';
import typeCastsTest from './tests/typeCasts.test';
let workspaceTest = () => {};
let ssoTest = () => {};
@ -39,6 +40,7 @@ function restTests() {
formulaTests();
ssoTest();
cloudOrgTest();
typeCastsTest();
// Enable for dashboard feature
// widgetTest();

133
packages/nocodb/tests/unit/rest/tests/typeCasts.test.ts

@ -0,0 +1,133 @@
import 'mocha';
import { UITypes } from 'nocodb-sdk';
import { expect } from 'chai';
import init from '../../init';
import { createProject } from '../../factory/base';
import { createTable } from '../../factory/table';
import { createBulkRows, rowMixedValue } from '../../factory/row';
import { updateColumn } from '../../factory/column';
import type Model from '../../../../src/models/Model';
import type Base from '../../../../src/models/Base';
import type Column from '../../../../src/models/Column';
const TEST_TYPES = [
UITypes.LongText,
UITypes.Attachment,
UITypes.Checkbox,
UITypes.MultiSelect,
UITypes.SingleSelect,
UITypes.Date,
UITypes.Year,
UITypes.Time,
UITypes.PhoneNumber,
UITypes.GeoData,
UITypes.Email,
UITypes.URL,
UITypes.Number,
UITypes.Decimal,
UITypes.Currency,
UITypes.Percent,
UITypes.Duration,
UITypes.Rating,
UITypes.Count,
UITypes.DateTime,
UITypes.Geometry,
UITypes.JSON,
UITypes.User,
];
async function setup(context, base: Base, type: UITypes) {
const table = await createTable(context, base, {
table_name: 'sampleTable',
title: 'sampleTable',
columns: [
{
column_name: 'Test',
title: 'Test',
uidt: type,
},
],
});
const column = (await table.getColumns({
workspace_id: base.fk_workspace_id,
base_id: base.id,
}))[0];
const rowAttributes = [];
for (let i = 0; i < 100; i++) {
const row = {
Title: rowMixedValue(column, i),
};
rowAttributes.push(row);
}
await createBulkRows(context, {
base,
table,
values: rowAttributes,
});
return { table, column };
}
function typeCastTests() {
let context;
let base: Base;
beforeEach(async function () {
context = await init();
base = await createProject(context);
});
describe('Single Line Text to ', () => {
let table: Model;
let column: Column;
beforeEach(async function () {
const data = await setup(context, base, UITypes.SingleLineText);
table = data.table;
column = data.column;
});
for (const type of TEST_TYPES)
it(type, async () => {
if (context.dbConfig.client !== 'pg') return;
const updatedColumn = await updateColumn(context, {
table,
column: column,
attr: {
...column,
uidt: type,
},
});
expect(updatedColumn.uidt).to.equal(type);
});
});
describe('Convert to Single Line Text from ', () => {
for (const type of TEST_TYPES)
it(type, async () => {
if (context.dbConfig.client !== 'pg') return;
const data = await setup(context, base, type);
const updatedColumn = await updateColumn(context, {
table: data.table,
column: data.column,
attr: {
...data.column,
uidt: UITypes.SingleLineText,
},
});
expect(updatedColumn.uidt).to.equal(UITypes.SingleLineText);
});
});
}
export default function () {
describe('Field type conversion ', typeCastTests);
}

34
tests/playwright/pages/Dashboard/Grid/Column/index.ts

@ -61,7 +61,6 @@ export class ColumnPageObject extends BasePage {
formula = '',
qrCodeValueColumnTitle = '',
barcodeValueColumnTitle = '',
barcodeFormat = '',
childTable = '',
childColumn = '',
relationType = '',
@ -282,7 +281,7 @@ export class ColumnPageObject extends BasePage {
})
.click();
await this.save();
await this.save({ isUpdated: true });
}
async changeReferencedColumnForBarcode({ titleOfReferencedColumn }: { titleOfReferencedColumn: string }) {
@ -293,7 +292,7 @@ export class ColumnPageObject extends BasePage {
})
.click();
await this.save();
await this.save({ isUpdated: true });
}
async changeBarcodeFormat({ barcodeFormatName }: { barcodeFormatName: string }) {
@ -304,7 +303,7 @@ export class ColumnPageObject extends BasePage {
})
.click();
await this.save();
await this.save({ isUpdated: true });
}
async delete({ title }: { title: string }) {
@ -423,13 +422,26 @@ export class ColumnPageObject extends BasePage {
await expect(this.grid.get().locator(`th[data-title="${title}"]`)).toHaveCount(0);
}
async save({ isUpdated }: { isUpdated?: boolean } = {}) {
await this.waitForResponse({
uiAction: async () => await this.get().locator('button[data-testid="nc-field-modal-submit-btn"]').click(),
requestUrlPathToMatch: 'api/v1/db/data/noco/',
httpMethodsToMatch: ['GET'],
responseJsonMatcher: json => json['pageInfo'],
});
async save({ isUpdated, typeChange }: { isUpdated?: boolean; typeChange?: boolean } = {}) {
// if type is changed, then we need to click the update button during the warning popup
if (!typeChange) {
const buttonText = isUpdated ? 'Update' : 'Save';
await this.waitForResponse({
uiAction: async () => await this.get().locator(`button:has-text("${buttonText}")`).click(),
requestUrlPathToMatch: 'api/v1/db/data/noco/',
httpMethodsToMatch: ['GET'],
responseJsonMatcher: json => json['pageInfo'],
});
} else {
await this.get().locator('button:has-text("Update Field")').click();
// click on update button on warning popup
await this.waitForResponse({
uiAction: async () => await this.rootPage.locator('button:has-text("Update")').click(),
requestUrlPathToMatch: 'api/v1/db/data/noco/',
httpMethodsToMatch: ['GET'],
responseJsonMatcher: json => json['pageInfo'],
});
}
await this.verifyToast({
message: isUpdated ? 'Column updated' : 'Column created',

4
tests/playwright/tests/db/columns/columnUserSelect.spec.ts

@ -167,7 +167,7 @@ test.describe('User single select', () => {
// Convert User field column to SingleLineText
await grid.column.openEdit({ title: 'User copy' });
await grid.column.selectType({ type: 'SingleLineText' });
await grid.column.save({ isUpdated: true });
await grid.column.save({ isUpdated: true, typeChange: true });
// Verify converted column content
for (let i = 0; i <= 4; i++) {
@ -541,7 +541,7 @@ test.describe('User multiple select', () => {
// Convert User field column to SingleLineText
await grid.column.openEdit({ title: 'User copy' });
await grid.column.selectType({ type: 'SingleLineText' });
await grid.column.save({ isUpdated: true });
await grid.column.save({ isUpdated: true, typeChange: true });
// Verify converted column content
counter = 1;

2
tests/playwright/tests/db/general/tableColumnOperation.spec.ts

@ -22,7 +22,7 @@ test.describe('Table Column Operations', () => {
await grid.column.openEdit({ title: 'column_name_a' });
await grid.column.fillTitle({ title: 'column_name_b' });
await grid.column.selectType({ type: 'LongText' });
await grid.column.save({ isUpdated: true });
await grid.column.save({ isUpdated: true, typeChange: true });
await grid.column.verify({ title: 'column_name_b' });
await grid.column.delete({ title: 'column_name_b' });

2
tests/playwright/tests/db/views/viewCalendar.spec.ts

@ -439,7 +439,7 @@ test.describe('Calendar View', () => {
selectType: true,
});
await dashboard.grid.column.save({ isUpdated: true });
await dashboard.grid.column.save({ isUpdated: true, typeChange: true });
await dashboard.viewSidebar.createCalendarView({
title: 'Calendar',

2
tests/playwright/tests/db/views/viewKanban.spec.ts

@ -49,7 +49,7 @@ test.describe('View', () => {
});
count = count + 1;
}
await dashboard.grid.column.save();
await dashboard.grid.column.save({ typeChange: true });
}
});

Loading…
Cancel
Save