Browse Source

chore: sync with develop

pull/5724/head
Wing-Kam Wong 2 years ago
parent
commit
7beccb57f4
  1. 10374
      package-lock.json
  2. 50
      packages/nc-gui/components/cell/DateTimePicker.vue
  3. 4
      packages/nc-gui/components/dashboard/TreeView.vue
  4. 7
      packages/nc-gui/components/dlg/ProjectDuplicate.vue
  5. 2
      packages/nc-gui/components/dlg/TableDuplicate.vue
  6. 7
      packages/nc-gui/components/smartsheet/Cell.vue
  7. 6
      packages/nc-gui/components/smartsheet/Grid.vue
  8. 6
      packages/nc-gui/components/virtual-cell/Formula.vue
  9. 2
      packages/nc-gui/components/virtual-cell/components/ItemChip.vue
  10. 2
      packages/nc-gui/components/virtual-cell/components/ListChildItems.vue
  11. 2
      packages/nc-gui/components/virtual-cell/components/ListItems.vue
  12. 3
      packages/nc-gui/composables/useMultiSelect/convertCellData.ts
  13. 58
      packages/nc-gui/composables/useMultiSelect/index.ts
  14. 2
      packages/nc-gui/lang/fr.json
  15. 6
      packages/nc-gui/lang/ru.json
  16. 8
      packages/nc-gui/lang/zh-Hans.json
  17. 2
      packages/nc-gui/lib/types.ts
  18. 1
      packages/nc-gui/nuxt-shim.d.ts
  19. 4
      packages/nc-gui/pages/index/index/index.vue
  20. 2
      packages/nc-gui/plugins/a.dayjs.ts
  21. 17
      packages/nc-gui/plugins/jobs.ts
  22. 7
      packages/nc-gui/store/project.ts
  23. 23
      packages/nc-gui/utils/cell.ts
  24. 8
      packages/nc-gui/utils/dateTimeUtils.ts
  25. 1
      packages/nc-gui/utils/index.ts
  26. 9
      packages/nocodb-sdk/src/lib/Api.ts
  27. 10
      packages/nocodb/src/app.module.ts
  28. 4
      packages/nocodb/src/cache/RedisCacheMgr.ts
  29. 19
      packages/nocodb/src/connection/connection.spec.ts
  30. 37
      packages/nocodb/src/connection/connection.ts
  31. 294
      packages/nocodb/src/db/BaseModelSqlv2.ts
  32. 14
      packages/nocodb/src/db/CustomKnex.ts
  33. 45
      packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
  34. 8
      packages/nocodb/src/db/functionMappings/mssql.ts
  35. 2
      packages/nocodb/src/db/functionMappings/mysql.ts
  36. 6
      packages/nocodb/src/db/functionMappings/sqlite.ts
  37. 12
      packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts
  38. 3
      packages/nocodb/src/index.ts
  39. 6
      packages/nocodb/src/init.ts
  40. 12
      packages/nocodb/src/main.ts
  41. 79
      packages/nocodb/src/meta/meta.service.ts
  42. 65
      packages/nocodb/src/models/Model.ts
  43. 4
      packages/nocodb/src/modules/datas/datas.module.ts
  44. 27
      packages/nocodb/src/modules/global/global.module.ts
  45. 65
      packages/nocodb/src/modules/global/init-meta-service.provider.ts
  46. 24
      packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts
  47. 15
      packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts
  48. 51
      packages/nocodb/src/modules/jobs/fallback/jobs.service.ts
  49. 48
      packages/nocodb/src/modules/jobs/jobs.gateway.ts
  50. 54
      packages/nocodb/src/modules/jobs/jobs.module.ts
  51. 59
      packages/nocodb/src/modules/jobs/jobs.service.ts
  52. 32
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts
  53. 40
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  54. 0
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/EntityMap.ts
  55. 0
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
  56. 4
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts
  57. 0
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/syncMap.ts
  58. 45
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts
  59. 12
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  60. 20
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts
  61. 40
      packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts
  62. 16
      packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts
  63. 107
      packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts
  64. 53
      packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts
  65. 98
      packages/nocodb/src/modules/jobs/redis/jobs.service.ts
  66. 5
      packages/nocodb/src/modules/metas/metas.module.ts
  67. 4
      packages/nocodb/src/modules/test/test.module.ts
  68. 4
      packages/nocodb/src/modules/users/users.module.ts
  69. 17
      packages/nocodb/src/plugins/storage/Local.ts
  70. 27
      packages/nocodb/src/schema/swagger.json
  71. 19
      packages/nocodb/src/services/app-init.service.spec.ts
  72. 79
      packages/nocodb/src/services/app-init.service.ts
  73. 7
      packages/nocodb/src/services/auth.service.ts
  74. 4
      packages/nocodb/src/services/projects.service.ts
  75. 4
      packages/nocodb/src/services/utils.service.ts
  76. 755
      packages/nocodb/src/utils/NcConfigFactory.ts
  77. 2
      packages/nocodb/src/utils/common/NcConnectionMgr.ts
  78. 2
      packages/nocodb/src/utils/common/NcConnectionMgrv2.ts
  79. 182
      packages/nocodb/src/utils/nc-config/NcConfig.ts
  80. 84
      packages/nocodb/src/utils/nc-config/constants.ts
  81. 324
      packages/nocodb/src/utils/nc-config/helpers.ts
  82. 4
      packages/nocodb/src/utils/nc-config/index.ts
  83. 39
      packages/nocodb/src/utils/nc-config/interfaces.ts
  84. 12
      packages/nocodb/tests/unit/TestDbMngr.ts
  85. 51
      packages/nocodb/tests/unit/model/tests/baseModelSql.test.ts
  86. 4
      packages/nocodb/tests/unit/rest/tests/tableRow.test.ts
  87. 4498
      tests/playwright/package-lock.json
  88. 3
      tests/playwright/package.json
  89. 9
      tests/playwright/pages/Dashboard/ExpandedForm/index.ts
  90. 25
      tests/playwright/pages/Dashboard/TreeView.ts
  91. 29
      tests/playwright/pages/Dashboard/common/Cell/DateTimeCell.ts
  92. 13
      tests/playwright/pages/Dashboard/common/Cell/index.ts
  93. 1132
      tests/playwright/tests/db/timezone.spec.ts
  94. 2
      tests/playwright/tests/db/viewGridShare.spec.ts
  95. 48
      tests/playwright/tests/utils/config.ts
  96. 12
      tests/playwright/tests/utils/general.ts

10374
package-lock.json generated

File diff suppressed because it is too large Load Diff

50
packages/nc-gui/components/cell/DateTimePicker.vue

@ -18,13 +18,14 @@ import {
interface Props {
modelValue?: string | null
isPk?: boolean
isUpdatedFromCopyNPaste: Record<string, boolean>
}
const { modelValue, isPk } = defineProps<Props>()
const { modelValue, isPk, isUpdatedFromCopyNPaste } = defineProps<Props>()
const emit = defineEmits(['update:modelValue'])
const { isMysql } = useProject()
const { isMssql, isXcdbBase } = useProject()
const { showNull } = useGlobal()
@ -44,6 +45,8 @@ const dateTimeFormat = $computed(() => {
return `${dateFormat} ${timeFormat}`
})
let localModelValue = modelValue ? dayjs(modelValue).utc().local() : undefined
let localState = $computed({
get() {
if (!modelValue) {
@ -55,7 +58,45 @@ let localState = $computed({
return undefined
}
const isXcDB = isXcdbBase(column.value.base_id)
// cater copy and paste
// when copying a datetime cell, the copied value would be local time
// when pasting a datetime cell, UTC (xcdb) will be saved in DB
// we convert back to local time
if (column.value.title! in (isUpdatedFromCopyNPaste ?? {})) {
localModelValue = dayjs(modelValue).utc().local()
return localModelValue
}
// ext db
if (!isXcDB) {
return /^\d+$/.test(modelValue) ? dayjs(+modelValue) : dayjs(modelValue)
}
if (isMssql(column.value.base_id)) {
// e.g. 2023-04-29T11:41:53.000Z
return dayjs(modelValue)
}
// if cdf is defined, that means the value is auto-generated
// hence, show the local time
if (column?.value?.cdf) {
return dayjs(modelValue).utc().local()
}
// if localModelValue is defined, show localModelValue instead
// localModelValue is set in setter below
if (localModelValue) {
const res = localModelValue
// resetting localModelValue here
// e.g. save in expanded form -> render the correct modelValue
localModelValue = undefined
return res
}
// empty cell - use modelValue in local time
return dayjs(modelValue).utc().local()
},
set(val?: dayjs.Dayjs) {
if (!val) {
@ -64,7 +105,10 @@ let localState = $computed({
}
if (val.isValid()) {
emit('update:modelValue', val?.format(isMysql(column.value.base_id) ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ'))
// setting localModelValue to cater NOW function in date picker
localModelValue = dayjs(val)
// send the payload in UTC format
emit('update:modelValue', dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ'))
}
},
})

4
packages/nc-gui/components/dashboard/TreeView.vue

@ -399,8 +399,8 @@ const duplicateTable = async (table: TableType) => {
const { close } = useDialog(resolveComponent('DlgTableDuplicate'), {
'modelValue': isOpen,
'table': table,
'onOk': async (jobData: { name: string; id: string }) => {
$jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string, data?: any) => {
'onOk': async (jobData: { id: string }) => {
$jobs.subscribe({ id: jobData.id }, undefined, async (status: string, data?: any) => {
if (status === JobStatus.COMPLETED) {
await loadTables()
const newTable = tables.value.find((el) => el.id === data?.result?.id)

7
packages/nc-gui/components/dlg/ProjectDuplicate.vue

@ -34,7 +34,12 @@ const isLoading = ref(false)
const _duplicate = async () => {
isLoading.value = true
try {
const jobData = await api.project.duplicate(props.project.id as string, optionsToExclude.value)
const jobData = await api.project.duplicate(props.project.id as string, {
options: optionsToExclude.value,
project: {
meta: props.project.meta,
},
})
props.onOk(jobData as any)
} catch (e: any) {
message.error(await extractSdkResponseErrorMsg(e))

2
packages/nc-gui/components/dlg/TableDuplicate.vue

@ -34,7 +34,7 @@ const isLoading = ref(false)
const _duplicate = async () => {
isLoading.value = true
try {
const jobData = await api.dbTable.duplicate(props.table.project_id!, props.table.id!, optionsToExclude.value)
const jobData = await api.dbTable.duplicate(props.table.project_id!, props.table.id!, { options: optionsToExclude.value })
props.onOk(jobData as any)
} catch (e: any) {
message.error(await extractSdkResponseErrorMsg(e))

7
packages/nc-gui/components/smartsheet/Cell.vue

@ -209,7 +209,12 @@ onUnmounted(() => {
<LazyCellMultiSelect v-else-if="isMultiSelect(column)" v-model="vModel" :row-index="props.rowIndex" />
<LazyCellDatePicker v-else-if="isDate(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
<LazyCellYearPicker v-else-if="isYear(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
<LazyCellDateTimePicker v-else-if="isDateTime(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
<LazyCellDateTimePicker
v-else-if="isDateTime(column, abstractType)"
v-model="vModel"
:is-pk="isPrimaryKey(column)"
:is-updated-from-copy-n-paste="currentRow.rowMeta.isUpdatedFromCopyNPaste"
/>
<LazyCellTimePicker v-else-if="isTime(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
<LazyCellRating v-else-if="isRating(column)" v-model="vModel" />
<LazyCellDuration v-else-if="isDuration(column)" v-model="vModel" />

6
packages/nc-gui/components/smartsheet/Grid.vue

@ -316,6 +316,12 @@ const {
return
}
// See DateTimePicker.vue for details
data.value[ctx.row].rowMeta.isUpdatedFromCopyNPaste = {
...data.value[ctx.row].rowMeta.isUpdatedFromCopyNPaste,
[ctx.updatedColumnTitle || columnObj.title]: true,
}
// update/save cell value
await updateOrSaveRow(rowObj, ctx.updatedColumnTitle || columnObj.title)
},

6
packages/nc-gui/components/virtual-cell/Formula.vue

@ -1,7 +1,7 @@
<script lang="ts" setup>
import type { ColumnType } from 'nocodb-sdk'
import type { Ref } from 'vue'
import { CellValueInj, ColumnInj, computed, handleTZ, inject, replaceUrlsWithLink, useProject } from '#imports'
import { CellValueInj, ColumnInj, computed, handleTZ, inject, renderValue, replaceUrlsWithLink, useProject } from '#imports'
// todo: column type doesn't have required property `error` - throws in typecheck
const column = inject(ColumnInj) as Ref<ColumnType & { colOptions: { error: any } }>
@ -10,7 +10,9 @@ const cellValue = inject(CellValueInj)
const { isPg } = useProject()
const result = computed(() => (isPg(column.value.base_id) ? handleTZ(cellValue?.value) : cellValue?.value))
const result = computed(() =>
isPg(column.value.base_id) ? renderValue(handleTZ(cellValue?.value)) : renderValue(cellValue?.value),
)
const urls = computed(() => replaceUrlsWithLink(result.value))

2
packages/nc-gui/components/virtual-cell/components/ItemChip.vue

@ -9,9 +9,9 @@ import {
inject,
isAttachment,
ref,
renderValue,
useExpandedFormDetached,
useLTARStoreOrThrow,
useUIPermission,
} from '#imports'
interface Props {

2
packages/nc-gui/components/virtual-cell/components/ListChildItems.vue

@ -13,10 +13,10 @@ import {
iconMap,
inject,
ref,
renderValue,
useLTARStoreOrThrow,
useSmartsheetRowStoreOrThrow,
useVModel,
watch,
} from '#imports'
const props = defineProps<{ modelValue?: boolean; cellValue: any; column: any }>()

2
packages/nc-gui/components/virtual-cell/components/ListItems.vue

@ -12,11 +12,11 @@ import {
inject,
isDrawerExist,
ref,
renderValue,
useLTARStoreOrThrow,
useSelectedCellKeyupListener,
useSmartsheetRowStoreOrThrow,
useVModel,
watch,
} from '#imports'
const props = defineProps<{ modelValue: boolean; column: any }>()

3
packages/nc-gui/composables/useMultiSelect/convertCellData.ts

@ -7,6 +7,7 @@ import { parseProp } from '#imports'
export default function convertCellData(
args: { from: UITypes; to: UITypes; value: any; column: ColumnType; appInfo: AppInfo },
isMysql = false,
isXcdbBase = false,
) {
const { from, to, value } = args
if (from === to && ![UITypes.Attachment, UITypes.Date, UITypes.DateTime, UITypes.Time, UITypes.Year].includes(to)) {
@ -42,7 +43,7 @@ export default function convertCellData(
if (!parsedDateTime.isValid()) {
throw new Error('Not a valid datetime value')
}
return parsedDateTime.format(dateFormat)
return parsedDateTime.utc().format('YYYY-MM-DD HH:mm:ssZ')
}
case UITypes.Time: {
let parsedTime = dayjs(value)

58
packages/nc-gui/composables/useMultiSelect/index.ts

@ -1,3 +1,4 @@
import dayjs from 'dayjs'
import type { MaybeRef } from '@vueuse/core'
import type { ColumnType, LinkToAnotherRecordType, TableType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isVirtualCol } from 'nocodb-sdk'
@ -7,6 +8,7 @@ import convertCellData from './convertCellData'
import type { Nullable, Row } from '~/lib'
import {
copyTable,
dateFormats,
extractPkFromRow,
extractSdkResponseErrorMsg,
isMac,
@ -14,6 +16,7 @@ import {
message,
reactive,
ref,
timeFormats,
unref,
useCopy,
useEventListener,
@ -52,7 +55,7 @@ export function useMultiSelect(
const { appInfo } = useGlobal()
const { isMysql } = useProject()
const { isMysql, isXcdbBase } = useProject()
let clipboardContext = $ref<{ value: any; uidt: UITypes } | null>(null)
@ -79,6 +82,20 @@ export function useMultiSelect(
activeCell.col = col
}
function constructDateTimeFormat(column: ColumnType) {
const dateFormat = constructDateFormat(column)
const timeFormat = constructTimeFormat(column)
return `${dateFormat} ${timeFormat}`
}
function constructDateFormat(column: ColumnType) {
return parseProp(column?.meta)?.date_format ?? dateFormats[0]
}
function constructTimeFormat(column: ColumnType) {
return parseProp(column?.meta)?.time_format ?? timeFormats[0]
}
async function copyValue(ctx?: Cell) {
try {
if (selectedRange.start !== null && selectedRange.end !== null && !selectedRange.isSingleCell()) {
@ -106,6 +123,43 @@ export function useMultiSelect(
if (typeof textToCopy === 'object') {
textToCopy = JSON.stringify(textToCopy)
}
if (columnObj.uidt === UITypes.Formula) {
textToCopy = textToCopy.replace(/\b(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2})\b/g, (d: string) => {
// TODO(timezone): retrieve the format from the corresponding column meta
// assume hh:mm at this moment
return dayjs(d).utc().local().format('YYYY-MM-DD HH:mm')
})
}
if (columnObj.uidt === UITypes.DateTime || columnObj.uidt === UITypes.Time) {
// remove `"`
// e.g. "2023-05-12T08:03:53.000Z" -> 2023-05-12T08:03:53.000Z
textToCopy = textToCopy.replace(/["']/g, '')
const isMySQL = isMysql(columnObj.base_id)
let d = dayjs(textToCopy)
if (!d.isValid()) {
// insert a datetime value, copy the value without refreshing
// e.g. textToCopy = 2023-05-12T03:49:25.000Z
// feed custom parse format
d = dayjs(textToCopy, isMySQL ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ')
}
// users can change the datetime format in UI
// `textToCopy` would be always in YYYY-MM-DD HH:mm:ss(Z / +xx:yy) format
// therefore, here we reformat to the correct datetime format based on the meta
textToCopy = d.format(
columnObj.uidt === UITypes.DateTime ? constructDateTimeFormat(columnObj) : constructTimeFormat(columnObj),
)
if (columnObj.uidt === UITypes.DateTime && !dayjs(textToCopy).isValid()) {
throw new Error('Invalid DateTime')
}
}
await copy(textToCopy)
message.success(t('msg.info.copiedToClipboard'))
}
@ -305,6 +359,7 @@ export function useMultiSelect(
appInfo: unref(appInfo),
},
isMysql(meta.value?.base_id),
isXcdbBase(meta.value?.base_id),
)
e.preventDefault()
@ -339,6 +394,7 @@ export function useMultiSelect(
appInfo: unref(appInfo),
},
isMysql(meta.value?.base_id),
isXcdbBase(meta.value?.base_id),
)
e.preventDefault()
syncCellData?.(activeCell)

2
packages/nc-gui/lang/fr.json

@ -580,7 +580,7 @@
"afterEnablePwd": "L’accès est restreint par un mot de passe",
"privateLink": "Cette vue est partagée avec un lien privé",
"privateLinkAdditionalInfo": "Les personnes ayant le lien privé peuvent voir uniquement les cellules visibles de cette vue",
"afterFormSubmitted": "Après que le formulaire ait été soumis",
"afterFormSubmitted": "Après que le formulaire a été soumis",
"apiOptions": "Accéder au projet via",
"submitAnotherForm": "Afficher le bouton \"Soumettre un autre formulaire\"",
"showBlankForm": "Montrer un formulaire vierge après 5 secondes",

6
packages/nc-gui/lang/ru.json

@ -385,10 +385,10 @@
"nextRecord": "Следующая запись",
"previousRecord": "Предыдущая запись",
"copyApiURL": "Скопируйте URL API",
"createTable": "Create New Table",
"createTable": "Создать новую таблицу",
"refreshTable": "Обновление таблицы",
"renameTable": "Rename Table",
"deleteTable": "Delete Table",
"renameTable": "Переименовать таблицу",
"deleteTable": "Удалить таблицу",
"addField": "Добавить новое поле в эту таблицу",
"setDisplay": "Установить как значение отображения",
"addRow": "Добавить новую строку",

8
packages/nc-gui/lang/zh-Hans.json

@ -68,7 +68,7 @@
"questions": "问题",
"reachOut": "联系我们",
"betaNote": "此功能仍在测试中。",
"moreInfo": "这里可以找到更多信息",
"moreInfo": "点击此处了解更多信息。",
"logs": "日志",
"groupingField": "分组字段",
"insertAfter": "在右侧插入列",
@ -385,10 +385,10 @@
"nextRecord": "下一条记录",
"previousRecord": "上一条纪录",
"copyApiURL": "复制 API 链接",
"createTable": "Create New Table",
"createTable": "创建新的表格",
"refreshTable": "刷新表格",
"renameTable": "Rename Table",
"deleteTable": "Delete Table",
"renameTable": "重命名表格",
"deleteTable": "删除表格",
"addField": "添加新字段",
"setDisplay": "设置为显示值",
"addRow": "添加新行",

2
packages/nc-gui/lib/types.ts

@ -60,6 +60,8 @@ export interface Row {
commentCount?: number
changed?: boolean
saving?: boolean
// use in datetime picker component
isUpdatedFromCopyNPaste?: Record<string, boolean>
}
}

1
packages/nc-gui/nuxt-shim.d.ts vendored

@ -18,7 +18,6 @@ declare module '#app/nuxt' {
job:
| {
id: string
name: string
}
| any,
subscribedCb?: () => void,

4
packages/nc-gui/pages/index/index/index.vue

@ -90,10 +90,10 @@ const duplicateProject = (project: ProjectType) => {
const { close } = useDialog(resolveComponent('DlgProjectDuplicate'), {
'modelValue': isOpen,
'project': project,
'onOk': async (jobData: { name: string; id: string }) => {
'onOk': async (jobData: { id: string }) => {
await loadProjects()
$jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string) => {
$jobs.subscribe({ id: jobData.id }, undefined, async (status: string) => {
if (status === JobStatus.COMPLETED) {
await loadProjects()
} else if (status === JobStatus.FAILED) {

2
packages/nc-gui/plugins/a.dayjs.ts

@ -4,6 +4,7 @@ import customParseFormat from 'dayjs/plugin/customParseFormat.js'
import duration from 'dayjs/plugin/duration.js'
import utc from 'dayjs/plugin/utc.js'
import weekday from 'dayjs/plugin/weekday.js'
import timezone from 'dayjs/plugin/timezone.js'
import { defineNuxtPlugin } from '#imports'
export default defineNuxtPlugin(() => {
@ -12,4 +13,5 @@ export default defineNuxtPlugin(() => {
extend(customParseFormat)
extend(duration)
extend(weekday)
extend(timezone)
})

17
packages/nc-gui/plugins/jobs.ts

@ -29,22 +29,22 @@ export default defineNuxtPlugin(async (nuxtApp) => {
await init(nuxtApp.$state.token.value)
}
const send = (name: string, data: any) => {
const send = (evt: string, data: any) => {
if (socket) {
const _id = messageIndex++
socket.emit(name, { _id, data })
socket.emit(evt, { _id, data })
return _id
}
}
const jobs = {
subscribe(
job: { id: string; name: string } | any,
job: { id: string } | any,
subscribedCb?: () => void,
statusCb?: (status: JobStatus, data?: any) => void,
logCb?: (data: { message: string }) => void,
) {
const logFn = (data: { id: string; name: string; data: { message: string } }) => {
const logFn = (data: { id: string; data: { message: string } }) => {
if (data.id === job.id) {
if (logCb) logCb(data.data)
}
@ -61,11 +61,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
const _id = send('subscribe', job)
const subscribeFn = (data: { _id: number; name: string; id: string }) => {
const subscribeFn = (data: { _id: number; id: string }) => {
if (data._id === _id) {
if (data.id !== job.id || data.name !== job.name) {
if (data.id !== job.id) {
job.id = data.id
job.name = data.name
}
if (subscribedCb) subscribedCb()
socket?.on('log', logFn)
@ -75,10 +74,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
}
socket?.on('subscribed', subscribeFn)
},
getStatus(name: string, id: string): Promise<string> {
getStatus(id: string): Promise<string> {
return new Promise((resolve) => {
if (socket) {
const _id = send('status', { name, id })
const _id = send('status', { id })
const tempFn = (data: any) => {
if (data._id === _id) {
resolve(data.status)

7
packages/nc-gui/store/project.ts

@ -82,6 +82,10 @@ export const useProject = defineStore('projectStore', () => {
return ['mysql', ClientType.MYSQL].includes(getBaseType(baseId))
}
function isSqlite(baseId?: string) {
return getBaseType(baseId) === ClientType.SQLITE
}
function isMssql(baseId?: string) {
return getBaseType(baseId) === 'mssql'
}
@ -91,7 +95,7 @@ export const useProject = defineStore('projectStore', () => {
}
function isXcdbBase(baseId?: string) {
return bases.value.find((base) => base.id === baseId)?.is_meta
return (bases.value.find((base) => base.id === baseId)?.is_meta as boolean) || false
}
const isSharedBase = computed(() => projectType === 'base')
@ -209,6 +213,7 @@ export const useProject = defineStore('projectStore', () => {
isMysql,
isMssql,
isPg,
isSqlite,
sqlUis,
isSharedBase,
loadProjectMetaInfo,

23
packages/nc-gui/utils/cell.ts

@ -1,5 +1,6 @@
import type { ColumnType } from 'nocodb-sdk'
import { UITypes } from 'nocodb-sdk'
import dayjs from 'dayjs'
export const dataTypeLow = (column: ColumnType) => column.dt?.toLowerCase()
export const isBoolean = (column: ColumnType, abstractType?: any) =>
@ -53,3 +54,25 @@ export const isManualSaved = (column: ColumnType) => [UITypes.Currency].includes
export const isPrimary = (column: ColumnType) => !!column.pv
export const isPrimaryKey = (column: ColumnType) => !!column.pk
// used for LTAR and Formula
export const renderValue = (result?: any) => {
if (!result || typeof result !== 'string') {
return result
}
// convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
// e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
result = result.replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, (d: string) => {
return dayjs(d).isValid() ? dayjs(d).format('YYYY-MM-DD HH:mm:ssZ') : d
})
// convert all date time values to local time
// the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
// or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
return result.replace(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g, (d: string) => {
// TODO(timezone): retrieve the format from the corresponding column meta
// assume HH:mm at this moment
return dayjs(d).isValid() ? dayjs(d).format('YYYY-MM-DD HH:mm') : d
})
}

8
packages/nc-gui/utils/dateTimeUtils.ts

@ -1,7 +1,13 @@
import dayjs from 'dayjs'
export const timeAgo = (date: any) => {
return dayjs.utc(date).fromNow()
if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(date)) {
// if there is no timezone info, consider as UTC
// e.g. 2023-01-01 08:00:00 (MySQL)
date += '+00:00'
}
// show in local time
return dayjs(date).fromNow()
}
export const dateFormats = [

1
packages/nc-gui/utils/index.ts

@ -23,3 +23,4 @@ export * from './browserUtils'
export * from './geoDataUtils'
export * from './mimeTypeUtils'
export * from './parseUtils'
export * from './cell'

9
packages/nocodb-sdk/src/lib/Api.ts

@ -4032,9 +4032,12 @@ export class Api<
baseDuplicate: (
projectId: IdType,
data: {
options?: {
excludeData?: boolean;
excludeViews?: boolean;
excludeHooks?: boolean;
};
project?: object;
},
baseId?: IdType,
params: RequestParams = {}
@ -4078,9 +4081,12 @@ export class Api<
duplicate: (
projectId: IdType,
data: {
options?: {
excludeData?: boolean;
excludeViews?: boolean;
excludeHooks?: boolean;
};
project?: object;
},
params: RequestParams = {}
) =>
@ -5118,8 +5124,11 @@ export class Api<
projectId: IdType,
tableId: IdType,
data: {
options?: {
excludeData?: boolean;
excludeViews?: boolean;
excludeHooks?: boolean;
};
},
params: RequestParams = {}
) =>

10
packages/nocodb/src/app.module.ts

@ -1,6 +1,5 @@
import { Module, RequestMethod } from '@nestjs/common';
import { APP_FILTER } from '@nestjs/core';
import { BullModule } from '@nestjs/bull';
import { EventEmitterModule as NestJsEventEmitter } from '@nestjs/event-emitter';
import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter';
import { GlobalMiddleware } from './middlewares/global/global.middleware';
@ -17,7 +16,6 @@ import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.str
import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy';
import { MetasModule } from './modules/metas/metas.module';
import { JobsModule } from './modules/jobs/jobs.module';
import { AppInitService } from './services/app-init.service';
import type { MiddlewareConsumer } from '@nestjs/common';
@Module({
@ -30,13 +28,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
EventEmitterModule,
JobsModule,
NestJsEventEmitter.forRoot(),
...(process.env['NC_REDIS_URL']
? [
BullModule.forRoot({
redis: process.env.NC_REDIS_URL,
}),
]
: []),
],
controllers: [],
providers: [
@ -49,7 +40,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
AuthTokenStrategy,
BaseViewStrategy,
HookHandlerService,
AppInitService,
],
})
export class AppModule {

4
packages/nocodb/src/cache/RedisCacheMgr.ts vendored

@ -12,8 +12,12 @@ export default class RedisCacheMgr extends CacheMgr {
constructor(config: any) {
super();
this.client = new Redis(config);
// avoid flushing db in worker container
if (process.env.NC_WORKER_CONTAINER !== 'true') {
// flush the existing db with selected key (Default: 0)
this.client.flushdb();
}
// TODO(cache): fetch orgs once it's implemented
const orgs = 'noco';

19
packages/nocodb/src/connection/connection.spec.ts

@ -1,19 +0,0 @@
import { Test } from '@nestjs/testing';
import { Connection } from './knex';
import type { TestingModule } from '@nestjs/testing';
describe('Knex', () => {
let provider: Connection;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [Connection],
}).compile();
provider = module.get<Connection>(Connection);
});
it('should be defined', () => {
expect(provider).toBeDefined();
});
});

37
packages/nocodb/src/connection/connection.ts

@ -1,37 +0,0 @@
import { Global, Injectable, Scope } from '@nestjs/common';
import { XKnex } from '../db/CustomKnex';
import NcConfigFactory from '../utils/NcConfigFactory';
import type * as knex from 'knex';
@Injectable({
scope: Scope.DEFAULT,
})
export class Connection {
public static knex: knex.Knex;
public static _config: any;
get knexInstance(): knex.Knex {
return Connection.knex;
}
get config(): knex.Knex {
return Connection._config;
}
// init metadb connection
static async init(): Promise<void> {
Connection._config = await NcConfigFactory.make();
if (!Connection.knex) {
Connection.knex = XKnex({
...this._config.meta.db,
useNullAsDefault: true,
});
}
}
// init metadb connection
async init(): Promise<void> {
return await Connection.init();
}
}

294
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -1,6 +1,9 @@
import autoBind from 'auto-bind';
import groupBy from 'lodash/groupBy';
import DataLoader from 'dataloader';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc.js';
import timezone from 'dayjs/plugin/timezone';
import { nocoExecute } from 'nc-help';
import {
AuditOperationSubTypes,
@ -16,8 +19,16 @@ import DOMPurify from 'isomorphic-dompurify';
import { v4 as uuidv4 } from 'uuid';
import { NcError } from '../helpers/catchError';
import getAst from '../helpers/getAst';
import { Audit, Column, Filter, Model, Project, Sort, View } from '../models';
import {
Audit,
Base,
Column,
Filter,
Model,
Project,
Sort,
View,
} from '../models';
import { sanitize, unsanitize } from '../helpers/sqlSanitize';
import {
COMPARISON_OPS,
@ -46,7 +57,10 @@ import type {
SelectOption,
} from '../models';
import type { Knex } from 'knex';
import type { SortType } from 'nocodb-sdk';
import type { BoolType, SortType } from 'nocodb-sdk';
dayjs.extend(utc);
dayjs.extend(timezone);
const GROUP_COL = '__nc_group_id';
@ -1607,6 +1621,67 @@ class BaseModelSqlv2 {
if (!checkColumnRequired(column, fields, extractPkAndPv)) continue;
switch (column.uidt) {
case UITypes.DateTime:
if (this.isMySQL) {
// MySQL stores timestamp in UTC but display in timezone
// To verify the timezone, run `SELECT @@global.time_zone, @@session.time_zone;`
// If it's SYSTEM, then the timezone is read from the configuration file
// if a timezone is set in a DB, the retrieved value would be converted to the corresponding timezone
// for example, let's say the global timezone is +08:00 in DB
// the value 2023-01-01 10:00:00 (UTC) would display as 2023-01-01 18:00:00 (UTC+8)
// our existing logic is based on UTC, during the query, we need to take the UTC value
// hence, we use CONVERT_TZ to convert back to UTC value
res[sanitize(column.title || column.column_name)] =
this.dbDriver.raw(
`CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`,
[
`${sanitize(alias || this.model.table_name)}.${
column.column_name
}`,
],
);
break;
} else if (this.isPg) {
// if there is no timezone info,
// convert to database timezone,
// then convert to UTC
if (
column.dt !== 'timestamp with time zone' &&
column.dt !== 'timestamptz'
) {
res[sanitize(column.title || column.column_name)] = this.dbDriver
.raw(
`?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`,
[
`${sanitize(alias || this.model.table_name)}.${
column.column_name
}`,
],
)
.wrap('(', ')');
break;
}
} else if (this.isMssql) {
// if there is no timezone info,
// convert to database timezone,
// then convert to UTC
if (column.dt !== 'datetimeoffset') {
res[sanitize(column.title || column.column_name)] =
this.dbDriver.raw(
`CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
[
`${sanitize(alias || this.model.table_name)}.${
column.column_name
}`,
],
);
break;
}
}
res[sanitize(column.title || column.column_name)] = sanitize(
`${alias || this.model.table_name}.${column.column_name}`,
);
break;
case 'LinkToAnotherRecord':
case 'Lookup':
break;
@ -1734,7 +1809,11 @@ class BaseModelSqlv2 {
await populatePk(this.model, data);
// todo: filter based on view
const insertObj = await this.model.mapAliasToColumn(data);
const insertObj = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
);
await this.validate(insertObj);
@ -1872,7 +1951,11 @@ class BaseModelSqlv2 {
async updateByPk(id, data, trx?, cookie?) {
try {
const updateObj = await this.model.mapAliasToColumn(data);
const updateObj = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
);
await this.validate(data);
@ -1920,6 +2003,16 @@ class BaseModelSqlv2 {
return this.getTnPath(this.model);
}
public get clientMeta() {
return {
isSqlite: this.isSqlite,
isMssql: this.isMssql,
isPg: this.isPg,
isMySQL: this.isMySQL,
// isSnowflake: this.isSnowflake,
};
}
get isSqlite() {
return this.clientType === 'sqlite3';
}
@ -1948,7 +2041,11 @@ class BaseModelSqlv2 {
// const driver = trx ? trx : await this.dbDriver.transaction();
try {
await populatePk(this.model, data);
const insertObj = await this.model.mapAliasToColumn(data);
const insertObj = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
);
let rowId = null;
const postInsertOps = [];
@ -2106,7 +2203,11 @@ class BaseModelSqlv2 {
: await Promise.all(
datas.map(async (d) => {
await populatePk(this.model, d);
return this.model.mapAliasToColumn(d);
return this.model.mapAliasToColumn(
d,
this.clientMeta,
this.dbDriver,
);
}),
);
@ -2170,7 +2271,11 @@ class BaseModelSqlv2 {
const updateDatas = raw
? datas
: await Promise.all(datas.map((d) => this.model.mapAliasToColumn(d)));
: await Promise.all(
datas.map((d) =>
this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver),
),
);
const prevData = [];
const newData = [];
@ -2222,7 +2327,11 @@ class BaseModelSqlv2 {
) {
try {
let count = 0;
const updateData = await this.model.mapAliasToColumn(data);
const updateData = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
);
await this.validate(updateData);
const pkValues = await this._extractPksValues(updateData);
if (pkValues) {
@ -2268,7 +2377,9 @@ class BaseModelSqlv2 {
let transaction;
try {
const deleteIds = await Promise.all(
ids.map((d) => this.model.mapAliasToColumn(d)),
ids.map((d) =>
this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver),
),
);
const deleted = [];
@ -3154,16 +3265,23 @@ class BaseModelSqlv2 {
} else {
query = sanitize(query);
}
return this.convertAttachmentType(
let data =
this.isPg || this.isSnowflake
? (await this.dbDriver.raw(query))?.rows
: query.slice(0, 6) === 'select' && !this.isMssql
? await this.dbDriver.from(
this.dbDriver.raw(query).wrap('(', ') __nc_alias'),
)
: await this.dbDriver.raw(query),
childTable,
);
: await this.dbDriver.raw(query);
// update attachment fields
data = this.convertAttachmentType(data, childTable);
// update date time fields
data = this.convertDateFormat(data, childTable);
return data;
}
private _convertAttachmentType(
@ -3195,7 +3313,153 @@ class BaseModelSqlv2 {
this._convertAttachmentType(attachmentColumns, d),
);
} else {
this._convertAttachmentType(attachmentColumns, data);
data = this._convertAttachmentType(attachmentColumns, data);
}
}
}
return data;
}
// TODO(timezone): retrieve the format from the corresponding column meta
private _convertDateFormat(
dateTimeColumns: Record<string, any>[],
d: Record<string, any>,
) {
if (!d) return d;
for (const col of dateTimeColumns) {
if (!d[col.title]) continue;
if (col.uidt === UITypes.Formula) {
if (!d[col.title] || typeof d[col.title] !== 'string') {
continue;
}
// remove milliseconds
if (this.isMySQL) {
d[col.title] = d[col.title].replace(/\.000000/g, '');
} else if (this.isMssql) {
d[col.title] = d[col.title].replace(/\.0000000 \+00:00/g, '');
}
if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.title])) {
// convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
// e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
d[col.title] = d[col.title].replace(
/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g,
(d: string) => {
if (!dayjs(d).isValid()) return d;
if (this.isSqlite) {
// e.g. DATEADD formula
return dayjs(d).utc().format('YYYY-MM-DD HH:mm:ssZ');
}
return dayjs(d).utc(true).format('YYYY-MM-DD HH:mm:ssZ');
},
);
continue;
}
// convert all date time values to utc
// the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
// or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
d[col.title] = d[col.title].replace(
/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g,
(d: string) => {
if (!dayjs(d).isValid()) {
return d;
}
if (this.isSqlite) {
// if there is no timezone info,
// we assume the input is on NocoDB server timezone
// then we convert to UTC from server timezone
// example: datetime without timezone
// we need to display 2023-04-27 10:00:00 (in HKT)
// we convert d (e.g. 2023-04-27 18:00:00) to utc, i.e. 2023-04-27 02:00:00+00:00
// if there is timezone info,
// we simply convert it to UTC
// example: datetime with timezone
// e.g. 2023-04-27 10:00:00+05:30 -> 2023-04-27 04:30:00+00:00
return dayjs(d)
.tz(Intl.DateTimeFormat().resolvedOptions().timeZone)
.utc()
.format('YYYY-MM-DD HH:mm:ssZ');
}
// set keepLocalTime to true if timezone info is not found
const keepLocalTime = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/g.test(
d,
);
return dayjs(d).utc(keepLocalTime).format('YYYY-MM-DD HH:mm:ssZ');
},
);
continue;
}
let keepLocalTime = true;
if (this.isSqlite) {
if (!col.cdf) {
if (
d[col.title].indexOf('-') === -1 &&
d[col.title].indexOf('+') === -1 &&
d[col.title].slice(-1) !== 'Z'
) {
// if there is no timezone info,
// we assume the input is on NocoDB server timezone
// then we convert to UTC from server timezone
// e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00
d[col.title] = dayjs(d[col.title])
.tz(Intl.DateTimeFormat().resolvedOptions().timeZone)
.utc()
.format('YYYY-MM-DD HH:mm:ssZ');
continue;
} else {
// otherwise, we convert from the given timezone to UTC
keepLocalTime = false;
}
}
}
if (
this.isPg &&
(col.dt === 'timestamp with time zone' || col.dt === 'timestamptz')
) {
// postgres - timezone already attached to input
// e.g. 2023-05-11 16:16:51+08:00
keepLocalTime = false;
}
if (d[col.title] instanceof Date) {
// e.g. MSSQL
// Wed May 10 2023 17:47:46 GMT+0800 (Hong Kong Standard Time)
keepLocalTime = false;
}
// e.g. 01.01.2022 10:00:00+05:30 -> 2022-01-01 04:30:00+00:00
// e.g. 2023-05-09 11:41:49 -> 2023-05-09 11:41:49+00:00
d[col.title] = dayjs(d[col.title])
// keep the local time
.utc(keepLocalTime)
// show the timezone even for Mysql
.format('YYYY-MM-DD HH:mm:ssZ');
}
return d;
}
private convertDateFormat(data: Record<string, any>, childTable?: Model) {
// Show the date time in UTC format in API response
// e.g. 2022-01-01 04:30:00+00:00
if (data) {
const dateTimeColumns = (
childTable ? childTable.columns : this.model.columns
).filter(
(c) => c.uidt === UITypes.DateTime || c.uidt === UITypes.Formula,
);
if (dateTimeColumns.length) {
if (Array.isArray(data)) {
data = data.map((d) => this._convertDateFormat(dateTimeColumns, d));
} else {
data = this._convertDateFormat(dateTimeColumns, data);
}
}
}

14
packages/nocodb/src/db/CustomKnex.ts

@ -1,12 +1,24 @@
import { Knex, knex } from 'knex';
import { SnowflakeClient } from 'nc-help';
import { types } from 'pg';
import pg, { types } from 'pg';
import dayjs from 'dayjs';
import Filter from '../models/Filter';
import type { FilterType } from 'nocodb-sdk';
import type { BaseModelSql } from './BaseModelSql';
// For the code, check out
// https://raw.githubusercontent.com/brianc/node-pg-types/master/lib/builtins.js
// override parsing date column to Date()
types.setTypeParser(1082, (val) => val);
// override timestamp
types.setTypeParser(1114, (val) => {
return dayjs(val).format('YYYY-MM-DD HH:mm:ss');
});
// override timestampz
types.setTypeParser(1184, (val) => {
return dayjs(val).format('YYYY-MM-DD HH:mm:ssZ');
});
const opMappingGen = {
eq: '=',

45
packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts

@ -537,6 +537,51 @@ async function _formulaQueryBuilder(
};
};
break;
case UITypes.DateTime:
if (knex.clientType().startsWith('mysql')) {
aliasToColumn[col.id] = async (): Promise<any> => {
return {
// convert from DB timezone to UTC
builder: knex.raw(
`CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`,
[col.column_name],
),
};
};
} else if (
knex.clientType() === 'pg' &&
col.dt !== 'timestamp with time zone' &&
col.dt !== 'timestamptz'
) {
aliasToColumn[col.id] = async (): Promise<any> => {
return {
// convert from DB timezone to UTC
builder: knex
.raw(
`?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`,
[col.column_name],
)
.wrap('(', ')'),
};
};
} else if (
knex.clientType() === 'mssql' &&
col.dt !== 'datetimeoffset'
) {
// convert from DB timezone to UTC
aliasToColumn[col.id] = async (): Promise<any> => {
return {
builder: knex.raw(
`CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
[col.column_name],
),
};
};
} else {
aliasToColumn[col.id] = () =>
Promise.resolve({ builder: col.column_name });
}
break;
default:
aliasToColumn[col.id] = () =>
Promise.resolve({ builder: col.column_name });

8
packages/nocodb/src/db/functionMappings/mssql.ts

@ -130,15 +130,15 @@ const mssql = {
)},
${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
(await fn(pt.arguments[0])).builder
}), 'yyyy-MM-dd HH:mm')
}), 'yyyy-MM-dd HH:mm:ss')
ELSE
FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace(
/["']/g,
'',
)},
${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${fn(
pt.arguments[0],
)}), 'yyyy-MM-dd')
${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
(await fn(pt.arguments[0])).builder
}), 'yyyy-MM-dd')
END${colAlias}`,
),
};

2
packages/nocodb/src/db/functionMappings/mysql.ts

@ -62,7 +62,7 @@ const mysql2 = {
DATE_FORMAT(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL
${(await fn(pt.arguments[1])).builder} ${String(
(await fn(pt.arguments[2])).builder,
).replace(/["']/g, '')}), '%Y-%m-%d %H:%i')
).replace(/["']/g, '')}), '%Y-%m-%d %H:%i:%s')
ELSE
DATE(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL
${(await fn(pt.arguments[1])).builder} ${String(

6
packages/nocodb/src/db/functionMappings/sqlite.ts

@ -95,9 +95,9 @@ const sqlite3 = {
builder: knex.raw(
`CASE
WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN
STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${
STRFTIME('%Y-%m-%dT%H:%M:%fZ', DATETIME(DATETIME(${
(await fn(pt.arguments[0])).builder
}, 'localtime'),
}, 'utc'),
${dateIN > 0 ? '+' : ''}${
(await fn(pt.arguments[1])).builder
} || ' ${String((await fn(pt.arguments[2])).builder).replace(
@ -105,7 +105,7 @@ const sqlite3 = {
'',
)}'))
ELSE
DATE(DATETIME(${(await fn(pt.arguments[0])).builder}, 'localtime'),
DATE(DATETIME(${(await fn(pt.arguments[0])).builder}),
${dateIN > 0 ? '+' : ''}${
(await fn(pt.arguments[1])).builder
} || ' ${String((await fn(pt.arguments[2])).builder).replace(

12
packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts

@ -10,7 +10,7 @@ import Debug from '../../util/Debug';
import Emit from '../../util/emit';
import Result from '../../util/Result';
import * as fileHelp from '../../util/file.help';
import NcConfigFactory from '../../../utils/NcConfigFactory';
import { getToolDir, NcConfig } from '../../../utils/nc-config';
import SqlMigrator from './SqlMigrator';
const evt = new Emit();
@ -39,7 +39,7 @@ export default class KnexMigrator extends SqlMigrator {
this.project_id = projectObj?.project_id;
this.project = projectObj?.config;
this.metaDb = projectObj?.metaDb;
this.toolDir = NcConfigFactory.getToolDir();
this.toolDir = getToolDir();
}
emit(data, _args?) {
@ -312,8 +312,12 @@ export default class KnexMigrator extends SqlMigrator {
if (exists) {
await this._readProjectJson(projJsonFilePath);
this.emit('Migrator for project initalised successfully');
} else if (NcConfigFactory.hasDbUrl()) {
this.project = await NcConfigFactory.make();
} else if (
Object.keys(process.env).some((envKey) =>
envKey.startsWith('NC_DB_URL'),
)
) {
this.project = await NcConfig.createByEnv();
} else {
args.type = args.type || 'sqlite';

3
packages/nocodb/src/index.ts

@ -1,6 +1,5 @@
import Noco from './Noco';
import NcConfigFactory from './utils/NcConfigFactory';
export default Noco;
export { Noco, NcConfigFactory };
export { Noco };

6
packages/nocodb/src/init.ts

@ -1,12 +1,12 @@
import { Connection } from './connection/connection';
import { MetaService } from './meta/meta.service';
import { NcConfig } from './utils/nc-config';
import Noco from './Noco';
// run upgrader
import NcUpgrader from './version-upgrader/NcUpgrader';
export default async () => {
await Connection.init();
Noco._ncMeta = new MetaService(new Connection());
const config = await NcConfig.createByEnv();
Noco._ncMeta = new MetaService(config);
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
};

12
packages/nocodb/src/main.ts

@ -5,12 +5,22 @@ import { AppModule } from './app.module';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
app.use(express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }));
if (process.env.NC_WORKER_CONTAINER !== 'true') {
app.use(
express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }),
);
app.use(
cors({
exposedHeaders: 'xc-db-response',
}),
);
await app.listen(8080);
} else {
if (!process.env.NC_REDIS_URL) {
throw new Error('NC_REDIS_URL is required');
}
process.env.NC_DISABLE_TELE = 'true';
await app.init();
}
}
bootstrap();

79
packages/nocodb/src/meta/meta.service.ts

@ -1,20 +1,18 @@
import {
Global,
Inject,
Injectable,
OnApplicationBootstrap,
OnModuleInit,
Optional,
} from '@nestjs/common';
import { Injectable, Optional } from '@nestjs/common';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
import timezone from 'dayjs/plugin/timezone';
import { customAlphabet } from 'nanoid';
import CryptoJS from 'crypto-js';
import { Connection } from '../connection/connection';
import Noco from '../Noco';
import NocoCache from '../cache/NocoCache';
import { XKnex } from '../db/CustomKnex';
import { NcConfig } from '../utils/nc-config';
import XcMigrationSourcev2 from './migrations/XcMigrationSourcev2';
import XcMigrationSource from './migrations/XcMigrationSource';
import type { Knex } from 'knex';
import type * as knex from 'knex';
dayjs.extend(utc);
dayjs.extend(timezone);
const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4);
@ -187,18 +185,38 @@ const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14);
@Injectable()
export class MetaService {
constructor(private metaConnection: Connection, @Optional() trx = null) {
private _knex: knex.Knex;
private _config: any;
constructor(config: NcConfig, @Optional() trx = null) {
this._config = config;
this._knex = XKnex({
...this._config.meta.db,
useNullAsDefault: true,
});
this.trx = trx;
}
get knexInstance(): knex.Knex {
return this._knex;
}
get config(): NcConfig {
return this._config;
}
public get connection() {
return this.trx ?? this.metaConnection.knexInstance;
return this.trx ?? this.knexInstance;
}
get knexConnection() {
return this.connection;
}
public get knex(): any {
return this.knexConnection;
}
public async metaGet(
project_id: string,
dbAlias: string,
@ -256,8 +274,8 @@ export class MetaService {
await this.knexConnection(target).insert({
...insertObj,
created_at: data?.created_at || this.knexConnection?.fn?.now(),
updated_at: data?.updated_at || this.knexConnection?.fn?.now(),
created_at: this.now(),
updated_at: this.now(),
});
return insertObj;
}
@ -539,9 +557,9 @@ export class MetaService {
return this.knexConnection(target).insert({
db_alias: dbAlias,
project_id,
created_at: this.knexConnection?.fn?.now(),
updated_at: this.knexConnection?.fn?.now(),
...data,
created_at: this.now(),
updated_at: this.now(),
});
}
@ -689,7 +707,7 @@ export class MetaService {
delete data.created_at;
query.update({ ...data, updated_at: this.knexConnection?.fn?.now() });
query.update({ ...data, updated_at: this.now() });
if (typeof idOrCondition !== 'object') {
query.where('id', idOrCondition);
} else if (idOrCondition) {
@ -753,7 +771,7 @@ export class MetaService {
});
// todo: tobe done
return new MetaService(this.metaConnection, trx);
return new MetaService(this.config, trx);
}
async metaReset(
@ -810,8 +828,8 @@ export class MetaService {
// todo: check project name used or not
await this.knexConnection('nc_projects').insert({
...project,
created_at: this.knexConnection?.fn?.now(),
updated_at: this.knexConnection?.fn?.now(),
created_at: this.now(),
updated_at: this.now(),
});
// todo
@ -1022,14 +1040,23 @@ export class MetaService {
.delete();
}
public get knex(): any {
return this.knexConnection;
}
private getNanoId() {
return nanoid();
}
private isMySQL(): boolean {
return (
this.connection.clientType() === 'mysql' ||
this.connection.clientType() === 'mysql2'
);
}
private now(): any {
return dayjs()
.utc()
.format(this.isMySQL() ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ');
}
public async audit(
project_id: string,
dbAlias: string,

65
packages/nocodb/src/models/Model.ts

@ -1,4 +1,5 @@
import { isVirtualCol, ModelTypes, UITypes, ViewTypes } from 'nocodb-sdk';
import dayjs from 'dayjs';
import { BaseModelSqlv2 } from '../db/BaseModelSqlv2';
import Noco from '../Noco';
import { parseMetaProp } from '../utils/modelUtils';
@ -15,6 +16,7 @@ import { sanitize } from '../helpers/sqlSanitize';
import { extractProps } from '../helpers/extractProps';
import Audit from './Audit';
import View from './View';
import Base from './Base';
import Column from './Column';
import type { BoolType, TableReqType, TableType } from 'nocodb-sdk';
import type { XKnex } from '../db/CustomKnex';
@ -458,8 +460,18 @@ export default class Model implements TableType {
return true;
}
async mapAliasToColumn(data) {
async mapAliasToColumn(
data,
clientMeta = {
isMySQL: false,
isSqlite: false,
isMssql: false,
isPg: false,
},
knex,
) {
const insertObj = {};
const base = await Base.get(this.base_id);
for (const col of await this.getColumns()) {
if (isVirtualCol(col)) continue;
let val =
@ -470,6 +482,57 @@ export default class Model implements TableType {
if (col.uidt === UITypes.Attachment && typeof val !== 'string') {
val = JSON.stringify(val);
}
if (col.uidt === UITypes.DateTime && dayjs(val).isValid()) {
const { isMySQL, isSqlite, isMssql, isPg } = clientMeta;
if (
val.indexOf('-') < 0 &&
val.indexOf('+') < 0 &&
val.slice(-1) !== 'Z'
) {
// if no timezone is given,
// then append +00:00 to make it as UTC
val += '+00:00';
}
if (isMySQL) {
// first convert the value to utc
// from UI
// e.g. 2022-01-01 20:00:00Z -> 2022-01-01 20:00:00
// from API
// e.g. 2022-01-01 20:00:00+08:00 -> 2022-01-01 12:00:00
// if timezone info is not found - considered as utc
// e.g. 2022-01-01 20:00:00 -> 2022-01-01 20:00:00
// if timezone info is found
// e.g. 2022-01-01 20:00:00Z -> 2022-01-01 20:00:00
// e.g. 2022-01-01 20:00:00+00:00 -> 2022-01-01 20:00:00
// e.g. 2022-01-01 20:00:00+08:00 -> 2022-01-01 12:00:00
// then we use CONVERT_TZ to convert that in the db timezone
val = knex.raw(`CONVERT_TZ(?, '+00:00', @@GLOBAL.time_zone)`, [
dayjs(val).utc().format('YYYY-MM-DD HH:mm:ss'),
]);
} else if (isSqlite) {
// convert to UTC
// e.g. 2022-01-01T10:00:00.000Z -> 2022-01-01 04:30:00+00:00
val = dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ');
} else if (isPg) {
// convert to UTC
// e.g. 2023-01-01T12:00:00.000Z -> 2023-01-01 12:00:00+00:00
// then convert to db timezone
val = knex.raw(`? AT TIME ZONE CURRENT_SETTING('timezone')`, [
dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ'),
]);
} else if (isMssql) {
// convert ot UTC
// e.g. 2023-05-10T08:49:32.000Z -> 2023-05-10 08:49:32-08:00
// then convert to db timezone
val = knex.raw(
`SWITCHOFFSET(CONVERT(datetimeoffset, ?), DATENAME(TzOffset, SYSDATETIMEOFFSET()))`,
[dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ')],
);
} else {
// e.g. 2023-01-01T12:00:00.000Z -> 2023-01-01 12:00:00+00:00
val = dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ');
}
}
insertObj[sanitize(col.column_name)] = val;
}
}

4
packages/nocodb/src/modules/datas/datas.module.ts

@ -27,6 +27,8 @@ import { PublicDatasService } from '../../services/public-datas.service';
}),
],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [
DatasController,
BulkDataAliasController,
DataAliasController,
@ -35,6 +37,8 @@ import { PublicDatasService } from '../../services/public-datas.service';
OldDatasController,
PublicDatasController,
PublicDatasExportController,
]
: []),
],
providers: [
DatasService,

27
packages/nocodb/src/modules/global/global.module.ts

@ -1,25 +1,18 @@
import { Global, Module } from '@nestjs/common';
import { ExtractJwt } from 'passport-jwt';
import {
AppInitService,
appInitServiceProvider,
} from '../../services/app-init.service';
import { SocketGateway } from '../../gateways/socket.gateway';
import { Connection } from '../../connection/connection';
import { GlobalGuard } from '../../guards/global/global.guard';
import { MetaService } from '../../meta/meta.service';
import Noco from '../../Noco';
import { JwtStrategy } from '../../strategies/jwt.strategy';
import { UsersService } from '../../services/users/users.service';
import Noco from '../../Noco';
import { InitMetaServiceProvider } from './init-meta-service.provider';
import type { Provider } from '@nestjs/common';
export const JwtStrategyProvider: Provider = {
provide: JwtStrategy,
useFactory: async (
usersService: UsersService,
appInitService: AppInitService,
) => {
const config = appInitService.appConfig;
useFactory: async (usersService: UsersService, metaService: MetaService) => {
const config = metaService.config;
await Noco.initJwt();
@ -34,29 +27,25 @@ export const JwtStrategyProvider: Provider = {
return new JwtStrategy(options, usersService);
},
inject: [UsersService, AppInitService],
inject: [UsersService, MetaService],
};
@Global()
@Module({
imports: [],
providers: [
appInitServiceProvider,
Connection,
MetaService,
InitMetaServiceProvider,
UsersService,
JwtStrategyProvider,
GlobalGuard,
SocketGateway,
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
],
exports: [
AppInitService,
Connection,
MetaService,
JwtStrategyProvider,
UsersService,
GlobalGuard,
SocketGateway,
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
],
})
export class GlobalModule {}

65
packages/nocodb/src/modules/global/init-meta-service.provider.ts

@ -0,0 +1,65 @@
import { T } from 'nc-help';
import { MetaService } from '../../meta/meta.service';
import Noco from '../../Noco';
import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
import NcUpgrader from '../../version-upgrader/NcUpgrader';
import NocoCache from '../../cache/NocoCache';
import getInstance from '../../utils/getInstance';
import initAdminFromEnv from '../../helpers/initAdminFromEnv';
import { User } from '../../models';
import { NcConfig, prepareEnv } from '../../utils/nc-config';
import type { Provider } from '@nestjs/common';
import type { IEventEmitter } from '../event-emitter/event-emitter.interface';
export const InitMetaServiceProvider: Provider = {
// initialize app,
// 1. init cache
// 2. init db connection and create if not exist
// 3. init meta and set to Noco
// 4. init jwt
// 5. init plugin manager
// 6. run upgrader
useFactory: async (eventEmitter: IEventEmitter) => {
// NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB
await prepareEnv();
const config = await NcConfig.createByEnv();
// set version
process.env.NC_VERSION = '0107004';
// init cache
await NocoCache.init();
// init meta service
const metaService = new MetaService(config);
await metaService.init();
// provide meta and config to Noco
Noco._ncMeta = metaService;
Noco.config = config;
Noco.eventEmitter = eventEmitter;
// init jwt secret
await Noco.initJwt();
// load super admin user from env if env is set
await initAdminFromEnv(metaService);
// init plugin manager
await NcPluginMgrv2.init(Noco.ncMeta);
await Noco.loadEEState();
// run upgrader
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
T.init({
instance: getInstance,
});
T.emit('evt_app_started', await User.count());
return metaService;
},
provide: MetaService,
inject: ['IEventEmitter'],
};

24
packages/nocodb/src/modules/jobs/fallback-queue.service.ts → packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@ -1,12 +1,12 @@
import { Injectable } from '@nestjs/common';
import PQueue from 'p-queue';
import Emittery from 'emittery';
import { JobStatus, JobTypes } from '../../interface/Jobs';
import { DuplicateProcessor } from './export-import/duplicate.processor';
import { JobStatus, JobTypes } from '../../../interface/Jobs';
import { DuplicateProcessor } from '../jobs/export-import/duplicate.processor';
import { AtImportProcessor } from '../jobs/at-import/at-import.processor';
import { JobsEventService } from './jobs-event.service';
import { AtImportProcessor } from './at-import/at-import.processor';
interface Job {
export interface Job {
id: string;
name: string;
status: string;
@ -27,16 +27,12 @@ export class QueueService {
private readonly atImportProcessor: AtImportProcessor,
) {
this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => {
const job = this.queueMemory.find(
(job) => job.id === data.job.id && job.name === data.job.name,
);
const job = this.queueMemory.find((job) => job.id === data.job.id);
job.status = JobStatus.ACTIVE;
this.jobsEventService.onActive.apply(this.jobsEventService, [job as any]);
});
this.emitter.on(JobStatus.COMPLETED, (data: { job: Job; result: any }) => {
const job = this.queueMemory.find(
(job) => job.id === data.job.id && job.name === data.job.name,
);
const job = this.queueMemory.find((job) => job.id === data.job.id);
job.status = JobStatus.COMPLETED;
this.jobsEventService.onCompleted.apply(this.jobsEventService, [
job,
@ -46,9 +42,7 @@ export class QueueService {
this.removeJob(job);
});
this.emitter.on(JobStatus.FAILED, (data: { job: Job; error: Error }) => {
const job = this.queueMemory.find(
(job) => job.id === data.job.id && job.name === data.job.name,
);
const job = this.queueMemory.find((job) => job.id === data.job.id);
job.status = JobStatus.FAILED;
this.jobsEventService.onFailed.apply(this.jobsEventService, [
job,
@ -126,9 +120,7 @@ export class QueueService {
// remove job from memory
private removeJob(job: Job) {
const fIndex = this.queueMemory.findIndex(
(q) => q.id === job.id && q.name === job.name,
);
const fIndex = this.queueMemory.findIndex((q) => q.id === job.id);
if (fIndex) {
this.queueMemory.splice(fIndex, 1);
}

15
packages/nocodb/src/modules/jobs/jobs-event.service.ts → packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts

@ -7,7 +7,7 @@ import {
import { Job } from 'bull';
import boxen from 'boxen';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
@Processor(JOBS_QUEUE)
export class JobsEventService {
@ -16,7 +16,6 @@ export class JobsEventService {
@OnQueueActive()
onActive(job: Job) {
this.eventEmitter.emit(JobEvents.STATUS, {
name: job.name,
id: job.id.toString(),
status: JobStatus.ACTIVE,
});
@ -26,7 +25,7 @@ export class JobsEventService {
onFailed(job: Job, error: Error) {
console.error(
boxen(
`---- !! JOB FAILED !! ----\nname: ${job.name}\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
{
padding: 1,
borderStyle: 'double',
@ -36,7 +35,6 @@ export class JobsEventService {
);
this.eventEmitter.emit(JobEvents.STATUS, {
name: job.name,
id: job.id.toString(),
status: JobStatus.FAILED,
data: {
@ -50,7 +48,6 @@ export class JobsEventService {
@OnQueueCompleted()
onCompleted(job: Job, data: any) {
this.eventEmitter.emit(JobEvents.STATUS, {
name: job.name,
id: job.id.toString(),
status: JobStatus.COMPLETED,
data: {
@ -58,12 +55,4 @@ export class JobsEventService {
},
});
}
sendLog(job: Job, data: { message: string }) {
this.eventEmitter.emit(JobEvents.LOG, {
name: job.name,
id: job.id.toString(),
data,
});
}
}

51
packages/nocodb/src/modules/jobs/fallback/jobs.service.ts

@ -0,0 +1,51 @@
import { Injectable } from '@nestjs/common';
import { JobStatus } from '../../../interface/Jobs';
import { QueueService } from './fallback-queue.service';
@Injectable()
export class JobsService {
constructor(private readonly fallbackQueueService: QueueService) {}
async add(name: string, data: any) {
return this.fallbackQueueService.add(name, data);
}
async jobStatus(jobId: string) {
return await (
await this.fallbackQueueService.getJob(jobId)
).status;
}
async jobList() {
return await this.fallbackQueueService.getJobs([
JobStatus.ACTIVE,
JobStatus.WAITING,
JobStatus.DELAYED,
JobStatus.PAUSED,
]);
}
async getJobWithData(data: any) {
const jobs = await this.fallbackQueueService.getJobs([
// 'completed',
JobStatus.WAITING,
JobStatus.ACTIVE,
JobStatus.DELAYED,
// 'failed',
JobStatus.PAUSED,
]);
const job = jobs.find((j) => {
for (const key in data) {
if (j.data[key]) {
if (j.data[key] !== data[key]) return false;
} else {
return false;
}
}
return true;
});
return job;
}
}

48
packages/nocodb/src/modules/jobs/jobs.gateway.ts

@ -9,10 +9,10 @@ import { Server, Socket } from 'socket.io';
import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
import { AuthGuard } from '@nestjs/passport';
import { OnEvent } from '@nestjs/event-emitter';
import { Inject } from '@nestjs/common';
import { JobEvents } from '../../interface/Jobs';
import { JobsService } from './jobs.service';
import type { JobStatus } from '../../interface/Jobs';
import type { OnModuleInit } from '@nestjs/common';
import type { JobStatus } from '../../interface/Jobs';
@WebSocketGateway({
cors: {
@ -23,7 +23,7 @@ import type { OnModuleInit } from '@nestjs/common';
namespace: 'jobs',
})
export class JobsGateway implements OnModuleInit {
constructor(private readonly jobsService: JobsService) {}
constructor(@Inject('JobsService') private readonly jobsService) {}
@WebSocketServer()
server: Server;
@ -43,34 +43,28 @@ export class JobsGateway implements OnModuleInit {
@SubscribeMessage('subscribe')
async subscribe(
@MessageBody()
body: { _id: number; data: { id: string; name: string } | any },
body: { _id: number; data: { id: string } | any },
@ConnectedSocket() client: Socket,
): Promise<void> {
const { _id, data } = body;
if (
Object.keys(data).every((k) => ['name', 'id'].includes(k)) &&
data?.name &&
data?.id
) {
const rooms = (await this.jobsService.jobList(data.name)).map(
(j) => `${j.name}-${j.id}`,
if (Object.keys(data).every((k) => ['id'].includes(k)) && data?.id) {
const rooms = (await this.jobsService.jobList()).map(
(j) => `jobs-${j.id}`,
);
const room = rooms.find((r) => r === `${data.name}-${data.id}`);
const room = rooms.find((r) => r === `jobs-${data.id}`);
if (room) {
client.join(`${data.name}-${data.id}`);
client.join(`jobs-${data.id}`);
client.emit('subscribed', {
_id,
name: data.name,
id: data.id,
});
}
} else {
const job = await this.jobsService.getJobWithData(data);
if (job) {
client.join(`${job.name}-${job.id}`);
client.join(`jobs-${job.id}`);
client.emit('subscribed', {
_id,
name: job.name,
id: job.id,
});
}
@ -79,42 +73,30 @@ export class JobsGateway implements OnModuleInit {
@SubscribeMessage('status')
async status(
@MessageBody() body: { _id: number; data: { id: string; name: string } },
@MessageBody() body: { _id: number; data: { id: string } },
@ConnectedSocket() client: Socket,
): Promise<void> {
const { _id, data } = body;
client.emit('status', {
_id,
id: data.id,
name: data.name,
status: await this.jobsService.jobStatus(data.id),
});
}
@OnEvent(JobEvents.STATUS)
async sendJobStatus(data: {
name: string;
id: string;
status: JobStatus;
data?: any;
}): Promise<void> {
this.server.to(`${data.name}-${data.id}`).emit('status', {
sendJobStatus(data: { id: string; status: JobStatus; data?: any }): void {
this.server.to(`jobs-${data.id}`).emit('status', {
id: data.id,
name: data.name,
status: data.status,
data: data.data,
});
}
@OnEvent(JobEvents.LOG)
async sendJobLog(data: {
name: string;
id: string;
data: { message: string };
}): Promise<void> {
this.server.to(`${data.name}-${data.id}`).emit('log', {
sendJobLog(data: { id: string; data: { message: string } }): void {
this.server.to(`jobs-${data.id}`).emit('log', {
id: data.id,
name: data.name,
data: data.data,
});
}

54
packages/nocodb/src/modules/jobs/jobs.module.ts

@ -4,35 +4,59 @@ import { GlobalModule } from '../global/global.module';
import { DatasModule } from '../datas/datas.module';
import { MetasModule } from '../metas/metas.module';
import { JOBS_QUEUE } from '../../interface/Jobs';
import { JobsService } from './jobs.service';
import { ExportService } from './export-import/export.service';
import { ImportService } from './export-import/import.service';
import { DuplicateController } from './export-import/duplicate.controller';
import { DuplicateProcessor } from './export-import/duplicate.processor';
import { ExportService } from './jobs/export-import/export.service';
import { ImportService } from './jobs/export-import/import.service';
import { AtImportController } from './jobs/at-import/at-import.controller';
import { AtImportProcessor } from './jobs/at-import/at-import.processor';
import { DuplicateController } from './jobs/export-import/duplicate.controller';
import { DuplicateProcessor } from './jobs/export-import/duplicate.processor';
import { JobsLogService } from './jobs/jobs-log.service';
import { JobsGateway } from './jobs.gateway';
import { QueueService } from './fallback-queue.service';
import { JobsEventService } from './jobs-event.service';
import { AtImportController } from './at-import/at-import.controller';
import { AtImportProcessor } from './at-import/at-import.processor';
// Redis
import { JobsService } from './redis/jobs.service';
import { JobsRedisService } from './redis/jobs-redis.service';
import { JobsEventService } from './redis/jobs-event.service';
// Fallback
import { JobsService as FallbackJobsService } from './fallback/jobs.service';
import { QueueService as FallbackQueueService } from './fallback/fallback-queue.service';
import { JobsEventService as FallbackJobsEventService } from './fallback/jobs-event.service';
@Module({
imports: [
GlobalModule,
DatasModule,
MetasModule,
...(process.env.NC_REDIS_URL
? [
BullModule.forRoot({
url: process.env.NC_REDIS_URL,
}),
BullModule.registerQueue({
name: JOBS_QUEUE,
}),
]
: []),
],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [DuplicateController, AtImportController]
: []),
],
controllers: [DuplicateController, AtImportController],
providers: [
QueueService,
JobsGateway,
JobsService,
JobsEventService,
DuplicateProcessor,
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [JobsGateway] : []),
...(process.env.NC_REDIS_URL
? [JobsRedisService, JobsEventService]
: [FallbackQueueService, FallbackJobsEventService]),
{
provide: 'JobsService',
useClass: process.env.NC_REDIS_URL ? JobsService : FallbackJobsService,
},
JobsLogService,
ExportService,
ImportService,
DuplicateProcessor,
AtImportProcessor,
],
})

59
packages/nocodb/src/modules/jobs/jobs.service.ts

@ -1,59 +0,0 @@
import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
import { QueueService } from './fallback-queue.service';
@Injectable()
export class JobsService {
public activeQueue;
constructor(
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
private readonly fallbackQueueService: QueueService,
) {
this.activeQueue = this.fallbackQueueService;
/* process.env.NC_REDIS_URL
? this.jobsQueue
: this.fallbackQueueService;
*/
}
async jobStatus(jobId: string) {
return await (await this.activeQueue.getJob(jobId)).getState();
}
async jobList(jobType: string) {
return (
await this.activeQueue.getJobs([
JobStatus.ACTIVE,
JobStatus.WAITING,
JobStatus.DELAYED,
JobStatus.PAUSED,
])
).filter((j) => j.name === jobType);
}
async getJobWithData(data: any) {
const jobs = await this.activeQueue.getJobs([
// 'completed',
JobStatus.WAITING,
JobStatus.ACTIVE,
JobStatus.DELAYED,
// 'failed',
JobStatus.PAUSED,
]);
const job = jobs.find((j) => {
for (const key in data) {
if (j.data[key]) {
if (j.data[key] !== data[key]) return false;
} else {
return false;
}
}
return true;
});
return job;
}
}

32
packages/nocodb/src/modules/jobs/at-import/at-import.controller.ts → packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts

@ -1,30 +1,36 @@
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
import { GlobalGuard } from '../../../guards/global/global.guard';
import { ExtractProjectIdMiddleware } from '../../../middlewares/extract-project-id/extract-project-id.middleware';
import { SyncSource } from '../../../models';
import { NcError } from '../../../helpers/catchError';
import { JobsService } from '../jobs.service';
import { JobTypes } from '../../../interface/Jobs';
import {
Controller,
HttpCode,
Inject,
Post,
Request,
UseGuards,
} from '@nestjs/common';
import { GlobalGuard } from '../../../../guards/global/global.guard';
import { ExtractProjectIdMiddleware } from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
import { SyncSource } from '../../../../models';
import { NcError } from '../../../../helpers/catchError';
import { JobTypes } from '../../../../interface/Jobs';
@Controller()
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
export class AtImportController {
constructor(private readonly jobsService: JobsService) {}
constructor(@Inject('JobsService') private readonly jobsService) {}
@Post('/api/v1/db/meta/import/airtable')
@HttpCode(200)
async importAirtable(@Request() req) {
const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
const job = await this.jobsService.add(JobTypes.AtImport, {
...req.body,
});
return { id: job.id, name: job.name };
return { id: job.id };
}
@Post('/api/v1/db/meta/syncs/:syncId/trigger')
@HttpCode(200)
async triggerSync(@Request() req) {
const jobs = await this.jobsService.jobList(JobTypes.AtImport);
const jobs = await this.jobsService.jobList();
const fnd = jobs.find((j) => j.data.syncId === req.params.syncId);
if (fnd) {
@ -44,7 +50,7 @@ export class AtImportController {
baseURL = `http://localhost:${process.env.PORT || 8080}`;
}
const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
const job = await this.jobsService.add(JobTypes.AtImport, {
syncId: req.params.syncId,
...(syncSource?.details || {}),
projectId: syncSource.project_id,
@ -54,7 +60,7 @@ export class AtImportController {
user: user,
});
return { id: job.id, name: job.name };
return { id: job.id };
}
@Post('/api/v1/db/meta/syncs/:syncId/abort')

40
packages/nocodb/src/modules/jobs/at-import/at-import.processor.ts → packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@ -9,23 +9,23 @@ import utc from 'dayjs/plugin/utc';
import tinycolor from 'tinycolor2';
import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
import extractRolesObj from '../../../utils/extractRolesObj';
import { AttachmentsService } from '../../../services/attachments.service';
import { ColumnsService } from '../../../services/columns.service';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { FiltersService } from '../../../services/filters.service';
import { FormColumnsService } from '../../../services/form-columns.service';
import { GalleriesService } from '../../../services/galleries.service';
import { GridsService } from '../../../services/grids.service';
import { ProjectUsersService } from '../../../services/project-users/project-users.service';
import { ProjectsService } from '../../../services/projects.service';
import { SortsService } from '../../../services/sorts.service';
import { TablesService } from '../../../services/tables.service';
import { ViewColumnsService } from '../../../services/view-columns.service';
import { ViewsService } from '../../../services/views.service';
import { FormsService } from '../../../services/forms.service';
import { JobsEventService } from '../jobs-event.service';
import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
import extractRolesObj from '../../../../utils/extractRolesObj';
import { AttachmentsService } from '../../../../services/attachments.service';
import { ColumnsService } from '../../../../services/columns.service';
import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import { FiltersService } from '../../../../services/filters.service';
import { FormColumnsService } from '../../../../services/form-columns.service';
import { GalleriesService } from '../../../../services/galleries.service';
import { GridsService } from '../../../../services/grids.service';
import { ProjectUsersService } from '../../../../services/project-users/project-users.service';
import { ProjectsService } from '../../../../services/projects.service';
import { SortsService } from '../../../../services/sorts.service';
import { TablesService } from '../../../../services/tables.service';
import { ViewColumnsService } from '../../../../services/view-columns.service';
import { ViewsService } from '../../../../services/views.service';
import { FormsService } from '../../../../services/forms.service';
import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
import { JobsLogService } from '../jobs-log.service';
import FetchAT from './helpers/fetchAT';
import { importData, importLTARData } from './helpers/readAndProcessData';
import EntityMap from './helpers/EntityMap';
@ -99,7 +99,7 @@ export class AtImportProcessor {
private readonly viewColumnsService: ViewColumnsService,
private readonly sortsService: SortsService,
private readonly bulkDataAliasService: BulkDataAliasService,
private readonly jobsEventService: JobsEventService,
private readonly jobsLogService: JobsLogService,
) {}
@Process(JobTypes.AtImport)
@ -135,11 +135,11 @@ export class AtImportProcessor {
};
const logBasic = (log) => {
this.jobsEventService.sendLog(job, { message: log });
this.jobsLogService.sendLog(job, { message: log });
};
const logDetailed = (log) => {
if (debugMode) this.jobsEventService.sendLog(job, { message: log });
if (debugMode) this.jobsLogService.sendLog(job, { message: log });
};
const perfStats = [];

0
packages/nocodb/src/modules/jobs/at-import/helpers/EntityMap.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/EntityMap.ts

0
packages/nocodb/src/modules/jobs/at-import/helpers/fetchAT.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

4
packages/nocodb/src/modules/jobs/at-import/helpers/readAndProcessData.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@ -1,8 +1,8 @@
/* eslint-disable no-async-promise-executor */
import { RelationTypes, UITypes } from 'nocodb-sdk';
import EntityMap from './EntityMap';
import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import type { TablesService } from '../../../../services/tables.service';
import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service';
import type { TablesService } from '../../../../../services/tables.service';
// @ts-ignore
import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { TableType } from 'nocodb-sdk';

0
packages/nocodb/src/modules/jobs/at-import/helpers/syncMap.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/syncMap.ts

45
packages/nocodb/src/modules/jobs/export-import/duplicate.controller.ts → packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@ -2,28 +2,28 @@ import {
Body,
Controller,
HttpCode,
Inject,
Param,
Post,
Request,
UseGuards,
} from '@nestjs/common';
import { ProjectStatus } from 'nocodb-sdk';
import { GlobalGuard } from '../../../guards/global/global.guard';
import { GlobalGuard } from '../../../../guards/global/global.guard';
import {
Acl,
ExtractProjectIdMiddleware,
} from '../../../middlewares/extract-project-id/extract-project-id.middleware';
import { ProjectsService } from '../../../services/projects.service';
import { Base, Model, Project } from '../../../models';
import { generateUniqueName } from '../../../helpers/exportImportHelpers';
import { JobsService } from '../jobs.service';
import { JobTypes } from '../../../interface/Jobs';
} from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
import { ProjectsService } from '../../../../services/projects.service';
import { Base, Model, Project } from '../../../../models';
import { generateUniqueName } from '../../../../helpers/exportImportHelpers';
import { JobTypes } from '../../../../interface/Jobs';
@Controller()
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
export class DuplicateController {
constructor(
private readonly jobsService: JobsService,
@Inject('JobsService') private readonly jobsService,
private readonly projectsService: ProjectsService,
) {}
@ -35,10 +35,14 @@ export class DuplicateController {
@Param('projectId') projectId: string,
@Param('baseId') baseId?: string,
@Body()
body?: {
options?: {
excludeData?: boolean;
excludeViews?: boolean;
excludeHooks?: boolean;
};
// override duplicated project
project?: any;
},
) {
const project = await Project.get(projectId);
@ -63,22 +67,26 @@ export class DuplicateController {
);
const dupProject = await this.projectsService.projectCreate({
project: { title: uniqueTitle, status: ProjectStatus.JOB },
project: {
title: uniqueTitle,
status: ProjectStatus.JOB,
...(body.project || {}),
},
user: { id: req.user.id },
});
const job = await this.jobsService.activeQueue.add(JobTypes.DuplicateBase, {
const job = await this.jobsService.add(JobTypes.DuplicateBase, {
projectId: project.id,
baseId: base.id,
dupProjectId: dupProject.id,
options,
options: body.options || {},
req: {
user: req.user,
clientIp: req.clientIp,
},
});
return { id: job.id, name: job.name };
return { id: job.id };
}
@Post('/api/v1/db/meta/duplicate/:projectId/table/:modelId')
@ -89,10 +97,12 @@ export class DuplicateController {
@Param('projectId') projectId: string,
@Param('modelId') modelId?: string,
@Body()
body?: {
options?: {
excludeData?: boolean;
excludeViews?: boolean;
excludeHooks?: boolean;
};
},
) {
const project = await Project.get(projectId);
@ -116,21 +126,18 @@ export class DuplicateController {
models.map((p) => p.title),
);
const job = await this.jobsService.activeQueue.add(
JobTypes.DuplicateModel,
{
const job = await this.jobsService.add(JobTypes.DuplicateModel, {
projectId: project.id,
baseId: base.id,
modelId: model.id,
title: uniqueTitle,
options,
options: body.options || {},
req: {
user: req.user,
clientIp: req.clientIp,
},
},
);
});
return { id: job.id, name: job.name };
return { id: job.id };
}
}

12
packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts → packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -4,12 +4,12 @@ import { Job } from 'bull';
import papaparse from 'papaparse';
import { UITypes } from 'nocodb-sdk';
import { Logger } from '@nestjs/common';
import { Base, Column, Model, Project } from '../../../models';
import { ProjectsService } from '../../../services/projects.service';
import { findWithIdentifier } from '../../../helpers/exportImportHelpers';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
import { elapsedTime, initTime } from '../helpers';
import { Base, Column, Model, Project } from '../../../../models';
import { ProjectsService } from '../../../../services/projects.service';
import { findWithIdentifier } from '../../../../helpers/exportImportHelpers';
import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
import { elapsedTime, initTime } from '../../helpers';
import { ExportService } from './export.service';
import { ImportService } from './import.service';

20
packages/nocodb/src/modules/jobs/export-import/export.service.ts → packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -2,19 +2,19 @@ import { Readable } from 'stream';
import { UITypes, ViewTypes } from 'nocodb-sdk';
import { unparse } from 'papaparse';
import { Injectable, Logger } from '@nestjs/common';
import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers';
import NcConnectionMgrv2 from '../../../../utils/common/NcConnectionMgrv2';
import { getViewAndModelByAliasOrId } from '../../../datas/helpers';
import {
clearPrefix,
generateBaseIdMap,
} from '../../../helpers/exportImportHelpers';
import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
import { NcError } from '../../../helpers/catchError';
import { Base, Hook, Model, Project } from '../../../models';
import { DatasService } from '../../../services/datas.service';
import { elapsedTime, initTime } from '../helpers';
import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2';
import type { View } from '../../../models';
} from '../../../../helpers/exportImportHelpers';
import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
import { NcError } from '../../../../helpers/catchError';
import { Base, Hook, Model, Project } from '../../../../models';
import { DatasService } from '../../../../services/datas.service';
import { elapsedTime, initTime } from '../../helpers';
import type { BaseModelSqlv2 } from '../../../../db/BaseModelSqlv2';
import type { View } from '../../../../models';
@Injectable()
export class ExportService {

40
packages/nocodb/src/modules/jobs/export-import/import.service.ts → packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@ -9,28 +9,28 @@ import {
reverseGet,
withoutId,
withoutNull,
} from '../../../helpers/exportImportHelpers';
import { NcError } from '../../../helpers/catchError';
import { Base, Column, Model, Project } from '../../../models';
import { TablesService } from '../../../services/tables.service';
import { ColumnsService } from '../../../services/columns.service';
import { FiltersService } from '../../../services/filters.service';
import { SortsService } from '../../../services/sorts.service';
import { ViewColumnsService } from '../../../services/view-columns.service';
import { GridColumnsService } from '../../../services/grid-columns.service';
import { FormColumnsService } from '../../../services/form-columns.service';
import { GridsService } from '../../../services/grids.service';
import { FormsService } from '../../../services/forms.service';
import { GalleriesService } from '../../../services/galleries.service';
import { KanbansService } from '../../../services/kanbans.service';
import { HooksService } from '../../../services/hooks.service';
import { ViewsService } from '../../../services/views.service';
import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { elapsedTime, initTime } from '../helpers';
} from '../../../../helpers/exportImportHelpers';
import { NcError } from '../../../../helpers/catchError';
import { Base, Column, Model, Project } from '../../../../models';
import { TablesService } from '../../../../services/tables.service';
import { ColumnsService } from '../../../../services/columns.service';
import { FiltersService } from '../../../../services/filters.service';
import { SortsService } from '../../../../services/sorts.service';
import { ViewColumnsService } from '../../../../services/view-columns.service';
import { GridColumnsService } from '../../../../services/grid-columns.service';
import { FormColumnsService } from '../../../../services/form-columns.service';
import { GridsService } from '../../../../services/grids.service';
import { FormsService } from '../../../../services/forms.service';
import { GalleriesService } from '../../../../services/galleries.service';
import { KanbansService } from '../../../../services/kanbans.service';
import { HooksService } from '../../../../services/hooks.service';
import { ViewsService } from '../../../../services/views.service';
import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import { elapsedTime, initTime } from '../../helpers';
import type { Readable } from 'stream';
import type { ViewCreateReqType } from 'nocodb-sdk';
import type { LinkToAnotherRecordColumn, User, View } from '../../../models';
import type { LinkToAnotherRecordColumn, User, View } from '../../../../models';
@Injectable()
export class ImportService {

16
packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts

@ -0,0 +1,16 @@
import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { JobEvents } from '../../../interface/Jobs';
import type { Job } from 'bull';
@Injectable()
export class JobsLogService {
constructor(private eventEmitter: EventEmitter2) {}
sendLog(job: Job, data: { message: string }) {
this.eventEmitter.emit(JobEvents.LOG, {
id: job.id.toString(),
data,
});
}
}

107
packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts

@ -0,0 +1,107 @@
import {
OnQueueActive,
OnQueueCompleted,
OnQueueFailed,
Processor,
} from '@nestjs/bull';
import { Job } from 'bull';
import boxen from 'boxen';
import { EventEmitter2, OnEvent } from '@nestjs/event-emitter';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
import { JobsRedisService } from './jobs-redis.service';
@Processor(JOBS_QUEUE)
export class JobsEventService {
constructor(
private jobsRedisService: JobsRedisService,
private eventEmitter: EventEmitter2,
) {}
@OnQueueActive()
onActive(job: Job) {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
cmd: JobEvents.STATUS,
id: job.id.toString(),
status: JobStatus.ACTIVE,
});
} else {
this.eventEmitter.emit(JobEvents.STATUS, {
id: job.id.toString(),
status: JobStatus.ACTIVE,
});
}
}
@OnQueueFailed()
onFailed(job: Job, error: Error) {
console.error(
boxen(
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
{
padding: 1,
borderStyle: 'double',
borderColor: 'yellow',
},
),
);
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
cmd: JobEvents.STATUS,
id: job.id.toString(),
status: JobStatus.FAILED,
data: {
error: {
message: error?.message,
},
},
});
} else {
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
this.eventEmitter.emit(JobEvents.STATUS, {
id: job.id.toString(),
status: JobStatus.FAILED,
data: {
error: {
message: error?.message,
},
},
});
}
}
@OnQueueCompleted()
onCompleted(job: Job, data: any) {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
cmd: JobEvents.STATUS,
id: job.id.toString(),
status: JobStatus.COMPLETED,
data: {
result: data,
},
});
} else {
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
this.eventEmitter.emit(JobEvents.STATUS, {
id: job.id.toString(),
status: JobStatus.COMPLETED,
data: {
result: data,
},
});
}
}
@OnEvent(JobEvents.LOG)
onLog(data: { id: string; data: { message: string } }) {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${data.id}`, {
cmd: JobEvents.LOG,
id: data.id,
data: data.data,
});
}
}
}

53
packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts

@ -0,0 +1,53 @@
import { Injectable } from '@nestjs/common';
import Redis from 'ioredis';
@Injectable()
export class JobsRedisService {
private redisClient: Redis;
private redisSubscriber: Redis;
private unsubscribeCallbacks: { [key: string]: () => void } = {};
constructor() {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.redisClient = new Redis(process.env.NC_REDIS_URL);
return;
}
this.redisSubscriber = new Redis(process.env.NC_REDIS_URL);
}
publish(channel: string, message: string | any) {
if (typeof message === 'string') {
this.redisClient.publish(channel, message);
} else {
try {
this.redisClient.publish(channel, JSON.stringify(message));
} catch (e) {
console.error(e);
}
}
}
subscribe(channel: string, callback: (message: any) => void) {
this.redisSubscriber.subscribe(channel);
const onMessage = (_channel, message) => {
try {
message = JSON.parse(message);
} catch (e) {}
callback(message);
};
this.redisSubscriber.on('message', onMessage);
this.unsubscribeCallbacks[channel] = () => {
this.redisSubscriber.unsubscribe(channel);
this.redisSubscriber.off('message', onMessage);
};
}
unsubscribe(channel: string) {
if (this.unsubscribeCallbacks[channel]) {
this.unsubscribeCallbacks[channel]();
delete this.unsubscribeCallbacks[channel];
}
}
}

98
packages/nocodb/src/modules/jobs/redis/jobs.service.ts

@ -0,0 +1,98 @@
import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
import { JobsRedisService } from './jobs-redis.service';
import type { OnModuleInit } from '@nestjs/common';
@Injectable()
export class JobsService implements OnModuleInit {
constructor(
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
private jobsRedisService: JobsRedisService,
private eventEmitter: EventEmitter2,
) {}
// pause primary instance queue
async onModuleInit() {
if (process.env.NC_WORKER_CONTAINER !== 'true') {
await this.jobsQueue.pause(true);
}
}
async add(name: string, data: any) {
// resume primary instance queue if there is no worker
const workerCount = (await this.jobsQueue.getWorkers()).length;
const localWorkerPaused = await this.jobsQueue.isPaused(true);
// if there is no worker and primary instance queue is paused, resume it
// if there is any worker and primary instance queue is not paused, pause it
if (workerCount < 1 && localWorkerPaused) {
await this.jobsQueue.resume(true);
} else if (workerCount > 0 && !localWorkerPaused) {
await this.jobsQueue.pause(true);
}
const job = await this.jobsQueue.add(name, data);
// subscribe to job events
this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => {
const cmd = data.cmd;
delete data.cmd;
switch (cmd) {
case JobEvents.STATUS:
this.eventEmitter.emit(JobEvents.STATUS, data);
if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) {
this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`);
}
break;
case JobEvents.LOG:
this.eventEmitter.emit(JobEvents.LOG, data);
break;
}
});
return job;
}
async jobStatus(jobId: string) {
const job = await this.jobsQueue.getJob(jobId);
if (job) {
return await job.getState();
}
}
async jobList() {
return await this.jobsQueue.getJobs([
JobStatus.ACTIVE,
JobStatus.WAITING,
JobStatus.DELAYED,
JobStatus.PAUSED,
]);
}
async getJobWithData(data: any) {
const jobs = await this.jobsQueue.getJobs([
// 'completed',
JobStatus.WAITING,
JobStatus.ACTIVE,
JobStatus.DELAYED,
// 'failed',
JobStatus.PAUSED,
]);
const job = jobs.find((j) => {
for (const key in data) {
if (j.data[key]) {
if (j.data[key] !== data[key]) return false;
} else {
return false;
}
}
return true;
});
return job;
}
}

5
packages/nocodb/src/modules/metas/metas.module.ts

@ -67,7 +67,6 @@ import { UtilsService } from '../../services/utils.service';
import { ViewColumnsService } from '../../services/view-columns.service';
import { ViewsService } from '../../services/views.service';
import { ApiDocsService } from '../../services/api-docs/api-docs.service';
import { EventEmitterModule } from '../event-emitter/event-emitter.module';
import { GlobalModule } from '../global/global.module';
import { ProjectUsersController } from '../../controllers/project-users.controller';
import { ProjectUsersService } from '../../services/project-users/project-users.service';
@ -83,6 +82,8 @@ import { ProjectUsersService } from '../../services/project-users/project-users.
GlobalModule,
],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [
ApiDocsController,
ApiTokensController,
AttachmentsController,
@ -115,6 +116,8 @@ import { ProjectUsersService } from '../../services/project-users/project-users.
SyncController,
SortsController,
SharedBasesController,
]
: []),
],
providers: [
ApiDocsService,

4
packages/nocodb/src/modules/test/test.module.ts

@ -2,6 +2,8 @@ import { Module } from '@nestjs/common';
import { TestController } from '../../controllers/test/test.controller';
@Module({
controllers: [TestController],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [TestController] : []),
],
})
export class TestModule {}

4
packages/nocodb/src/modules/users/users.module.ts

@ -10,7 +10,9 @@ import { UsersController } from '../../controllers/users/users.controller';
@Module({
imports: [GlobalModule, PassportModule],
controllers: [UsersController],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [UsersController] : []),
],
providers: [UsersService, GoogleStrategyProvider],
exports: [UsersService],
})

17
packages/nocodb/src/plugins/storage/Local.ts

@ -3,7 +3,7 @@ import path from 'path';
import { promisify } from 'util';
import mkdirp from 'mkdirp';
import axios from 'axios';
import NcConfigFactory from '../../utils/NcConfigFactory';
import { getToolDir } from '../../utils/nc-config';
import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
import type { Readable } from 'stream';
@ -11,7 +11,7 @@ export default class Local implements IStorageAdapterV2 {
constructor() {}
public async fileCreate(key: string, file: XcFile): Promise<any> {
const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const destPath = path.join(getToolDir(), ...key.split('/'));
try {
await mkdirp(path.dirname(destPath));
const data = await promisify(fs.readFile)(file.path);
@ -24,7 +24,7 @@ export default class Local implements IStorageAdapterV2 {
}
async fileCreateByUrl(key: string, url: string): Promise<any> {
const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const destPath = path.join(getToolDir(), ...key.split('/'));
return new Promise((resolve, reject) => {
axios
.get(url, {
@ -71,10 +71,7 @@ export default class Local implements IStorageAdapterV2 {
stream: Readable,
): Promise<void> {
return new Promise((resolve, reject) => {
const destPath = path.join(
NcConfigFactory.getToolDir(),
...key.split('/'),
);
const destPath = path.join(getToolDir(), ...key.split('/'));
try {
mkdirp(path.dirname(destPath)).then(() => {
const writableStream = fs.createWriteStream(destPath);
@ -89,12 +86,12 @@ export default class Local implements IStorageAdapterV2 {
}
public async fileReadByStream(key: string): Promise<Readable> {
const srcPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const srcPath = path.join(getToolDir(), ...key.split('/'));
return fs.createReadStream(srcPath, { encoding: 'utf8' });
}
public async getDirectoryList(key: string): Promise<string[]> {
const destDir = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const destDir = path.join(getToolDir(), ...key.split('/'));
return fs.promises.readdir(destDir);
}
@ -106,7 +103,7 @@ export default class Local implements IStorageAdapterV2 {
public async fileRead(filePath: string): Promise<any> {
try {
const fileData = await fs.promises.readFile(
path.join(NcConfigFactory.getToolDir(), ...filePath.split('/')),
path.join(getToolDir(), ...filePath.split('/')),
);
return fileData;
} catch (e) {

27
packages/nocodb/src/schema/swagger.json

@ -2131,6 +2131,9 @@
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"excludeData": {
@ -2147,6 +2150,12 @@
}
}
},
"project": {
"type": "object",
"required": false
}
}
},
"examples": {
"Example 1": {
"value": {
@ -2221,6 +2230,9 @@
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"excludeData": {
@ -2237,6 +2249,12 @@
}
}
},
"project": {
"type": "object",
"required": false
}
}
},
"examples": {
"Example 1": {
"value": {
@ -3976,6 +3994,9 @@
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"excludeData": {
@ -3985,6 +4006,12 @@
"excludeViews": {
"type": "boolean",
"required": false
},
"excludeHooks": {
"type": "boolean",
"required": false
}
}
}
}
},

19
packages/nocodb/src/services/app-init.service.spec.ts

@ -1,19 +0,0 @@
import { Test } from '@nestjs/testing';
import { AppInitService } from './app-init.service';
import type { TestingModule } from '@nestjs/testing';
describe('AppInitService', () => {
let service: AppInitService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [AppInitService],
}).compile();
service = module.get<AppInitService>(AppInitService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

79
packages/nocodb/src/services/app-init.service.ts

@ -1,79 +0,0 @@
import { T } from 'nc-help';
import NocoCache from '../cache/NocoCache';
import { Connection } from '../connection/connection';
import initAdminFromEnv from '../helpers/initAdminFromEnv';
import NcPluginMgrv2 from '../helpers/NcPluginMgrv2';
import { MetaService } from '../meta/meta.service';
import { User } from '../models';
import Noco from '../Noco';
import getInstance from '../utils/getInstance';
import NcConfigFactory from '../utils/NcConfigFactory';
import NcUpgrader from '../version-upgrader/NcUpgrader';
import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface';
import type { Provider } from '@nestjs/common';
export class AppInitService {
private readonly config: any;
constructor(config) {
this.config = config;
}
get appConfig(): any {
return this.config;
}
}
export const appInitServiceProvider: Provider = {
provide: AppInitService,
// initialize app,
// 1. init cache
// 2. init db connection and create if not exist
// 3. init meta and set to Noco
// 4. init jwt
// 5. init plugin manager
// 6. run upgrader
useFactory: async (
connection: Connection,
metaService: MetaService,
eventEmitter: IEventEmitter,
) => {
process.env.NC_VERSION = '0107004';
await NocoCache.init();
await connection.init();
await NcConfigFactory.metaDbCreateIfNotExist(connection.config);
await metaService.init();
// todo: remove
// temporary hack
Noco._ncMeta = metaService;
Noco.config = connection.config;
Noco.eventEmitter = eventEmitter;
// init jwt secret
await Noco.initJwt();
// load super admin user from env if env is set
await initAdminFromEnv(metaService);
// init plugin manager
await NcPluginMgrv2.init(Noco.ncMeta);
await Noco.loadEEState();
// run upgrader
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
T.init({
instance: getInstance,
});
T.emit('evt_app_started', await User.count());
// todo: move app config to app-init service
return new AppInitService(connection.config);
},
inject: [Connection, MetaService, 'IEventEmitter'],
};

7
packages/nocodb/src/services/auth.service.ts

@ -5,18 +5,13 @@ import * as bcrypt from 'bcryptjs';
import { v4 as uuidv4 } from 'uuid';
import Noco from '../Noco';
import { Connection } from '../connection/connection';
import { genJwt } from './users/helpers';
import { UsersService } from './users/users.service';
import type { CreateUserDto } from '../controllers/auth.controller';
@Injectable()
export class AuthService {
constructor(
private usersService: UsersService,
// private jwtService: JwtService,
private connection: Connection,
) {}
constructor(private usersService: UsersService) {}
async validateUser(email: string, pass: string): Promise<any> {
const user = await this.usersService.findOne(email);

4
packages/nocodb/src/services/projects.service.ts

@ -11,7 +11,7 @@ import syncMigration from '../helpers/syncMigration';
import { Project, ProjectUser } from '../models';
import Noco from '../Noco';
import extractRolesObj from '../utils/extractRolesObj';
import NcConfigFactory from '../utils/NcConfigFactory';
import { getToolDir } from '../utils/nc-config';
import type { ProjectUpdateReqType } from 'nocodb-sdk';
import type { ProjectReqType } from 'nocodb-sdk';
@ -96,7 +96,7 @@ export class ProjectsService {
// if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project
// each file will be named as nc_<random_id>.db
const fs = require('fs');
const toolDir = NcConfigFactory.getToolDir();
const toolDir = getToolDir();
const nanoidv2 = customAlphabet(
'1234567890abcdefghijklmnopqrstuvwxyz',
14,

4
packages/nocodb/src/services/utils.service.ts

@ -9,7 +9,7 @@ import { Project, User } from '../models';
import Noco from '../Noco';
import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
import { MetaTable } from '../utils/globals';
import NcConfigFactory from '../utils/NcConfigFactory';
import { jdbcToXcConfig } from '../utils/nc-config/helpers';
import { packageVersion } from '../utils/packageVersion';
const versionCache = {
@ -186,7 +186,7 @@ export class UtilsService {
}) {
const { url } = param.body;
try {
const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true);
const connectionConfig = jdbcToXcConfig(url);
return connectionConfig;
} catch (error) {
return NcError.internalServerError(

755
packages/nocodb/src/utils/NcConfigFactory.ts

@ -1,755 +0,0 @@
import fs from 'fs';
import { URL } from 'url';
import { promisify } from 'util';
import * as path from 'path';
import parseDbUrl from 'parse-database-url';
import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory';
// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory';
// import type {
// AuthConfig,
// DbConfig,
// MailerConfig,
// NcConfig,
// } from '../../interface/config';
// const {
// uniqueNamesGenerator,
// starWars,
// adjectives,
// animals,
// } = require('unique-names-generator');
type NcConfig = any;
type DbConfig = any;
const driverClientMapping = {
mysql: 'mysql2',
mariadb: 'mysql2',
postgres: 'pg',
postgresql: 'pg',
sqlite: 'sqlite3',
mssql: 'mssql',
};
const defaultClientPortMapping = {
mysql: 3306,
mysql2: 3306,
postgres: 5432,
pg: 5432,
mssql: 1433,
};
const defaultConnectionConfig: any = {
// https://github.com/knex/knex/issues/97
// timezone: process.env.NC_TIMEZONE || 'UTC',
dateStrings: true,
};
// default knex options
const defaultConnectionOptions = {
pool: {
min: 0,
max: 10,
},
};
const knownQueryParams = [
{
parameter: 'database',
aliases: ['d', 'db'],
},
{
parameter: 'password',
aliases: ['p'],
},
{
parameter: 'user',
aliases: ['u'],
},
{
parameter: 'title',
aliases: ['t'],
},
{
parameter: 'keyFilePath',
aliases: [],
},
{
parameter: 'certFilePath',
aliases: [],
},
{
parameter: 'caFilePath',
aliases: [],
},
{
parameter: 'ssl',
aliases: [],
},
{
parameter: 'options',
aliases: ['opt', 'opts'],
},
];
export default class NcConfigFactory {
public static async make(): Promise<any> {
await this.jdbcToXcUrl();
const ncConfig = new NcConfigFactory();
ncConfig.auth = {
jwt: {
secret: process.env.NC_AUTH_JWT_SECRET,
},
};
ncConfig.port = +(process?.env?.PORT ?? 8080);
ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev';
ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev';
// ncConfig.toolDir = this.getToolDir();
ncConfig.projectType =
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
if (ncConfig.meta?.db?.connection?.filename) {
ncConfig.meta.db.connection.filename = path.join(
this.getToolDir(),
ncConfig.meta.db.connection.filename,
);
}
if (process.env.NC_DB) {
ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
} else if (process.env.NC_DB_JSON) {
ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON);
} else if (process.env.NC_DB_JSON_FILE) {
const filePath = process.env.NC_DB_JSON_FILE;
if (!(await promisify(fs.exists)(filePath))) {
throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`);
}
const fileContent = await promisify(fs.readFile)(filePath, {
encoding: 'utf8',
});
ncConfig.meta.db = JSON.parse(fileContent);
}
if (process.env.NC_TRY) {
ncConfig.try = true;
ncConfig.meta.db = {
client: 'sqlite3',
connection: ':memory:',
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
} as any;
}
if (process.env.NC_PUBLIC_URL) {
ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
// ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
ncConfig.publicUrl = process.env.NC_PUBLIC_URL;
}
if (process.env.NC_DASHBOARD_URL) {
ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL;
}
return ncConfig;
}
public static getToolDir() {
return process.env.NC_TOOL_DIR || process.cwd();
}
public static hasDbUrl(): boolean {
return Object.keys(process.env).some((envKey) =>
envKey.startsWith('NC_DB_URL'),
);
}
public static makeFromUrls(urls: string[]): NcConfig {
const config = new NcConfigFactory();
// config.envs[process.env.NODE_ENV || 'dev'].db = [];
config.envs['_noco'].db = [];
for (const [i, url] of Object.entries(urls)) {
// config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i));
config.envs['_noco'].db.push(this.urlToDbConfig(url, i));
}
return config;
}
public static urlToDbConfig(
urlString: string,
key = '',
config?: NcConfigFactory,
type?: string,
): DbConfig {
const url = new URL(urlString);
let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) {
dbConfig = {
client: 'sqlite3',
connection: {
client: 'sqlite3',
connection: {
filename:
url.searchParams.get('d') || url.searchParams.get('database'),
},
database:
url.searchParams.get('d') || url.searchParams.get('database'),
},
} as any;
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', ''),
connection: {
...defaultConnectionConfig,
...parsedQuery,
host: url.hostname,
port: +url.port,
},
// pool: {
// min: 1,
// max: 1
// },
acquireConnectionTimeout: 600000,
} as any;
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
if (
url.searchParams.get('keyFilePath') &&
url.searchParams.get('certFilePath') &&
url.searchParams.get('caFilePath')
) {
dbConfig.connection.ssl = {
keyFilePath: url.searchParams.get('keyFilePath'),
certFilePath: url.searchParams.get('certFilePath'),
caFilePath: url.searchParams.get('caFilePath'),
};
}
}
if (config && !config.title) {
config.title =
url.searchParams.get('t') ||
url.searchParams.get('title') ||
this.generateRandomTitle();
}
Object.assign(dbConfig, {
meta: {
tn: 'nc_evolutions',
allSchemas:
!!url.searchParams.get('allSchemas') ||
!(url.searchParams.get('d') || url.searchParams.get('database')),
api: {
prefix: url.searchParams.get('apiPrefix') || '',
swagger: true,
type:
type ||
((url.searchParams.get('api') ||
url.searchParams.get('a')) as any) ||
'rest',
},
dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
metaTables: 'db',
migrations: {
disabled: false,
name: 'nc_evolutions',
},
},
});
return dbConfig;
}
private static generateRandomTitle(): string {
return ''; /*uniqueNamesGenerator({
dictionaries: [[starWars], [adjectives, animals]][
Math.floor(Math.random() * 2)
],
})
.toLowerCase()
.replace(/[ -]/g, '_');*/
}
static async metaUrlToDbConfig(urlString) {
const url = new URL(urlString);
let dbConfig;
if (url.protocol.startsWith('sqlite3')) {
const db = url.searchParams.get('d') || url.searchParams.get('database');
dbConfig = {
client: 'sqlite3',
connection: {
filename: db,
},
...(db === ':memory:'
? {
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
}
: {}),
};
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', ''),
connection: {
...defaultConnectionConfig,
...parsedQuery,
host: url.hostname,
port: +url.port,
},
acquireConnectionTimeout: 600000,
...(url.searchParams.has('search_path')
? {
searchPath: url.searchParams.get('search_path').split(','),
}
: {}),
};
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
}
url.searchParams.forEach((_value, key) => {
let value: any = _value;
if (value === 'true') {
value = true;
} else if (value === 'false') {
value = false;
} else if (/^\d+$/.test(value)) {
value = +value;
}
// todo: implement config read from JSON file or JSON env val read
if (
![
'password',
'p',
'database',
'd',
'user',
'u',
'search_path',
].includes(key)
) {
key.split('.').reduce((obj, k, i, arr) => {
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
}, dbConfig);
}
});
if (
dbConfig?.connection?.ssl &&
typeof dbConfig?.connection?.ssl === 'object'
) {
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
dbConfig.connection.ssl.ca = (
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
).toString();
delete dbConfig.connection.ssl.caFilePath;
}
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
dbConfig.connection.ssl.key = (
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
).toString();
delete dbConfig.connection.ssl.keyFilePath;
}
if (
dbConfig.connection.ssl.certFilePath &&
!dbConfig.connection.ssl.cert
) {
dbConfig.connection.ssl.cert = (
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
).toString();
delete dbConfig.connection.ssl.certFilePath;
}
}
return dbConfig;
}
public static async makeProjectConfigFromUrl(
url,
type?: string,
): Promise<NcConfig> {
const config = new NcConfigFactory();
const dbConfig = this.urlToDbConfig(url, '', config, type);
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
config.envs['_noco'].db.push(dbConfig);
if (process.env.NC_AUTH_ADMIN_SECRET) {
config.auth = {
masterKey: {
secret: process.env.NC_AUTH_ADMIN_SECRET,
},
};
} else if (process.env.NC_NO_AUTH) {
config.auth = {
disabled: true,
};
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
} else if (config?.envs?.['_noco']?.db?.[0]) {
config.auth = {
jwt: {
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
dbAlias:
process.env.NC_AUTH_JWT_DB_ALIAS ||
config.envs['_noco'].db[0].meta.dbAlias,
secret: process.env.NC_AUTH_JWT_SECRET,
},
};
}
if (process.env.NC_DB) {
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
}
if (process.env.NC_TRY) {
config.try = true;
config.meta.db = {
client: 'sqlite3',
connection: ':memory:',
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
} as any;
}
if (process.env.NC_MAILER) {
config.mailer = {
from: process.env.NC_MAILER_FROM,
options: {
host: process.env.NC_MAILER_HOST,
port: parseInt(process.env.NC_MAILER_PORT, 10),
secure: process.env.NC_MAILER_SECURE === 'true',
auth: {
user: process.env.NC_MAILER_USER,
pass: process.env.NC_MAILER_PASS,
},
},
};
}
if (process.env.NC_PUBLIC_URL) {
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
config.publicUrl = process.env.NC_PUBLIC_URL;
}
config.port = +(process?.env?.PORT ?? 8080);
// config.env = process.env?.NODE_ENV || 'dev';
// config.workingEnv = process.env?.NODE_ENV || 'dev';
config.env = '_noco';
config.workingEnv = '_noco';
config.toolDir = this.getToolDir();
config.projectType =
type ||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
'rest';
return config;
}
public static async makeProjectConfigFromConnection(
dbConnectionConfig: any,
type?: string,
): Promise<NcConfig> {
const config = new NcConfigFactory();
let dbConfig = dbConnectionConfig;
if (dbConfig.client === 'sqlite3') {
dbConfig = {
client: 'sqlite3',
connection: {
...dbConnectionConfig,
database: dbConnectionConfig.connection.filename,
},
};
}
// todo:
const key = '';
Object.assign(dbConfig, {
meta: {
tn: 'nc_evolutions',
api: {
prefix: '',
swagger: true,
type: type || 'rest',
},
dbAlias: `db${key}`,
metaTables: 'db',
migrations: {
disabled: false,
name: 'nc_evolutions',
},
},
});
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
config.envs['_noco'].db.push(dbConfig);
if (process.env.NC_AUTH_ADMIN_SECRET) {
config.auth = {
masterKey: {
secret: process.env.NC_AUTH_ADMIN_SECRET,
},
};
} else if (process.env.NC_NO_AUTH) {
config.auth = {
disabled: true,
};
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
} else if (config?.envs?.['_noco']?.db?.[0]) {
config.auth = {
jwt: {
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
dbAlias:
process.env.NC_AUTH_JWT_DB_ALIAS ||
config.envs['_noco'].db[0].meta.dbAlias,
secret: process.env.NC_AUTH_JWT_SECRET,
},
};
}
if (process.env.NC_DB) {
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
}
if (process.env.NC_TRY) {
config.try = true;
config.meta.db = {
client: 'sqlite3',
connection: ':memory:',
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
} as any;
}
if (process.env.NC_PUBLIC_URL) {
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
config.publicUrl = process.env.NC_PUBLIC_URL;
}
config.port = +(process?.env?.PORT ?? 8080);
// config.env = process.env?.NODE_ENV || 'dev';
// config.workingEnv = process.env?.NODE_ENV || 'dev';
config.env = '_noco';
config.workingEnv = '_noco';
config.toolDir = process.env.NC_TOOL_DIR || process.cwd();
config.projectType =
type ||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
'rest';
return config;
}
public static async metaDbCreateIfNotExist(args: NcConfig) {
if (args.meta?.db?.client === 'sqlite3') {
const metaSqlClient = await SqlClientFactory.create({
...args.meta.db,
connection: args.meta.db,
});
await metaSqlClient.createDatabaseIfNotExists({
database: args.meta.db?.connection?.filename,
});
} else {
const metaSqlClient = await SqlClientFactory.create(args.meta.db);
await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection);
await metaSqlClient.knex.destroy();
}
/* const dbPath = path.join(args.toolDir, 'xc.db')
const exists = fs.existsSync(dbPath);
if (!exists) {
const fd = fs.openSync(dbPath, "w");
fs.closeSync(fd);
}
*/
}
public version = '0.6';
public port: number;
public auth?: any;
public env: 'production' | 'dev' | 'test' | string;
public workingEnv: string;
public toolDir: string;
public envs: {
[p: string]: { db: DbConfig[]; api?: any; publicUrl?: string };
};
// public projectType: "rest" | "graphql" | "grpc";
public queriesFolder: string | string[] = '';
public seedsFolder: string | string[];
public title: string;
public publicUrl: string;
public projectType;
public meta = {
db: {
client: 'sqlite3',
connection: {
filename: 'noco.db',
},
},
};
public mailer: any;
public try = false;
public dashboardPath = '/dashboard';
constructor() {
this.envs = { _noco: { db: [] } };
}
public static async jdbcToXcUrl() {
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
const database_url = await promisify(fs.readFile)(
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
'utf-8',
);
process.env.NC_DB = this.extractXcUrlFromJdbc(database_url);
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
process.env.NC_DB = this.extractXcUrlFromJdbc(
process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
);
}
}
public static extractXcUrlFromJdbc(url: string, rtConfig = false) {
// drop the jdbc prefix
if (url.startsWith('jdbc:')) {
url = url.substring(5);
}
const config = parseDbUrl(url);
const parsedConfig: {
driver?: string;
host?: string;
port?: string;
database?: string;
user?: string;
password?: string;
ssl?: string;
} = {};
for (const [key, value] of Object.entries(config)) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedConfig[fnd.parameter] = value;
} else {
parsedConfig[key] = value;
}
}
if (!parsedConfig?.port)
parsedConfig.port =
defaultClientPortMapping[
driverClientMapping[parsedConfig.driver] || parsedConfig.driver
];
if (rtConfig) {
const { driver, ...connectionConfig } = parsedConfig;
const client = driverClientMapping[driver] || driver;
const avoidSSL = [
'localhost',
'127.0.0.1',
'host.docker.internal',
'172.17.0.1',
];
if (
client === 'pg' &&
!connectionConfig?.ssl &&
!avoidSSL.includes(connectionConfig.host)
) {
connectionConfig.ssl = 'true';
}
return {
client: client,
connection: {
...connectionConfig,
},
} as any;
}
const { driver, host, port, database, user, password, ...extra } =
parsedConfig;
const extraParams = [];
for (const [key, value] of Object.entries(extra)) {
extraParams.push(`${key}=${value}`);
}
const res = `${driverClientMapping[driver] || driver}://${host}${
port ? `:${port}` : ''
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
database ? `d=${database}&` : ''
}${extraParams.join('&')}`;
return res;
}
// public static initOneClickDeployment() {
// if (process.env.NC_ONE_CLICK) {
// const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL);
// process.env.NC_DB = url;
// }
// }
}
export { defaultConnectionConfig, defaultConnectionOptions };

2
packages/nocodb/src/utils/common/NcConnectionMgr.ts

@ -2,7 +2,7 @@ import fs from 'fs';
import { promisify } from 'util';
import SqlClientFactory from '../../db/sql-client/lib/SqlClientFactory';
import { XKnex } from '../../db/CustomKnex';
import { defaultConnectionConfig } from '../NcConfigFactory';
import { defaultConnectionConfig } from '../nc-config';
// import type { NcConfig } from '../../../interface/config';
import type { Knex } from 'knex';
// import type NcMetaIO from '../../meta/NcMetaIO';

2
packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

@ -3,7 +3,7 @@ import { XKnex } from '../../db/CustomKnex';
import {
defaultConnectionConfig,
defaultConnectionOptions,
} from '../NcConfigFactory';
} from '../nc-config';
import Noco from '../../Noco';
import type Base from '../../models/Base';

182
packages/nocodb/src/utils/nc-config/NcConfig.ts

@ -0,0 +1,182 @@
import * as path from 'path';
import fs from 'fs';
import { promisify } from 'util';
import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory';
import { getToolDir, metaUrlToDbConfig } from './helpers';
import { DriverClient } from './interfaces';
import type { DbConfig } from './interfaces';
export class NcConfig {
version: string;
meta: {
db: DbConfig;
} = {
db: {
client: DriverClient.SQLITE,
connection: {
filename: 'noco.db',
},
},
};
auth: {
jwt: {
secret: string;
options?: any;
};
};
// if this is true, port is not exposed
worker: boolean;
toolDir: string;
// exposed instance port
port: number;
// if this is true, use sqlite3 :memory: as meta db
try: boolean;
// optional
publicUrl?: string;
dashboardPath?: string;
// TODO what is this?
envs: any;
queriesFolder: string;
env: string;
workingEnv: string;
projectType: string;
private constructor() {}
public static async create(param: {
meta: {
metaUrl?: string;
metaJson?: string;
metaJsonFile?: string;
};
secret?: string;
port?: string | number;
tryMode?: boolean;
worker?: boolean;
dashboardPath?: string;
publicUrl?: string;
}): Promise<NcConfig> {
const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } =
param;
const ncConfig = new NcConfig();
ncConfig.auth = {
jwt: {
secret: secret,
},
};
ncConfig.port = +(port ?? 8080);
ncConfig.toolDir = getToolDir();
ncConfig.worker = worker ?? false;
ncConfig.env = '_noco';
ncConfig.workingEnv = '_noco';
ncConfig.projectType =
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
if (ncConfig.meta?.db?.connection?.filename) {
ncConfig.meta.db.connection.filename = path.join(
ncConfig.toolDir,
ncConfig.meta.db.connection.filename,
);
}
if (tryMode) {
ncConfig.try = true;
ncConfig.meta.db = {
client: DriverClient.SQLITE,
connection: ':memory:' as any,
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
};
} else {
if (meta?.metaUrl) {
ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl);
} else if (meta?.metaJson) {
ncConfig.meta.db = JSON.parse(meta.metaJson);
} else if (meta?.metaJsonFile) {
if (!(await promisify(fs.exists)(meta.metaJsonFile))) {
throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`);
}
const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, {
encoding: 'utf8',
});
ncConfig.meta.db = JSON.parse(fileContent);
}
}
if (publicUrl) {
ncConfig.envs['_noco'].publicUrl = publicUrl;
ncConfig.publicUrl = publicUrl;
}
if (dashboardPath) {
ncConfig.dashboardPath = dashboardPath;
}
try {
// make sure meta db exists
await ncConfig.metaDbCreateIfNotExist();
} catch (e) {
throw new Error(e);
}
return ncConfig;
}
public static async createByEnv(): Promise<NcConfig> {
return NcConfig.create({
meta: {
metaUrl: process.env.NC_DB,
metaJson: process.env.NC_DB_JSON,
metaJsonFile: process.env.NC_DB_JSON_FILE,
},
secret: process.env.NC_AUTH_JWT_SECRET,
port: process.env.NC_PORT,
tryMode: !!process.env.NC_TRY,
worker: !!process.env.NC_WORKER,
dashboardPath: process.env.NC_DASHBOARD_PATH,
publicUrl: process.env.NC_PUBLIC_URL,
});
}
private async metaDbCreateIfNotExist() {
if (this.meta?.db?.client === 'sqlite3') {
const metaSqlClient = await SqlClientFactory.create({
...this.meta.db,
connection: this.meta.db,
});
if (this.meta.db?.connection?.filename) {
await metaSqlClient.createDatabaseIfNotExists({
database: this.meta.db?.connection?.filename,
});
} else {
throw new Error('Configuration missing meta db connection');
}
} else {
const metaSqlClient = await SqlClientFactory.create(this.meta.db);
if (this.meta.db?.connection?.database) {
await metaSqlClient.createDatabaseIfNotExists(
(this.meta.db as any).connection,
);
await metaSqlClient.knex.destroy();
} else {
throw new Error('Configuration missing meta db connection');
}
}
}
}

84
packages/nocodb/src/utils/nc-config/constants.ts

@ -0,0 +1,84 @@
export const driverClientMapping = {
mysql: 'mysql2',
mariadb: 'mysql2',
postgres: 'pg',
postgresql: 'pg',
sqlite: 'sqlite3',
mssql: 'mssql',
};
export const defaultClientPortMapping = {
mysql: 3306,
mysql2: 3306,
postgres: 5432,
pg: 5432,
mssql: 1433,
};
export const defaultConnectionConfig: any = {
// https://github.com/knex/knex/issues/97
// timezone: process.env.NC_TIMEZONE || 'UTC',
dateStrings: true,
};
// default knex options
export const defaultConnectionOptions = {
pool: {
min: 0,
max: 10,
},
};
export const avoidSSL = [
'localhost',
'127.0.0.1',
'host.docker.internal',
'172.17.0.1',
];
export const knownQueryParams = [
{
parameter: 'database',
aliases: ['d', 'db'],
},
{
parameter: 'password',
aliases: ['p'],
},
{
parameter: 'user',
aliases: ['u'],
},
{
parameter: 'title',
aliases: ['t'],
},
{
parameter: 'keyFilePath',
aliases: [],
},
{
parameter: 'certFilePath',
aliases: [],
},
{
parameter: 'caFilePath',
aliases: [],
},
{
parameter: 'ssl',
aliases: [],
},
{
parameter: 'options',
aliases: ['opt', 'opts'],
},
];
export enum DriverClient {
MYSQL = 'mysql2',
MSSQL = 'mssql',
PG = 'pg',
SQLITE = 'sqlite3',
SNOWFLAKE = 'snowflake',
}

324
packages/nocodb/src/utils/nc-config/helpers.ts

@ -0,0 +1,324 @@
import fs from 'fs';
import { URL } from 'url';
import { promisify } from 'util';
import parseDbUrl from 'parse-database-url';
import {
avoidSSL,
defaultClientPortMapping,
defaultConnectionConfig,
defaultConnectionOptions,
driverClientMapping,
knownQueryParams,
} from './constants';
import { DriverClient } from './interfaces';
import type { Connection, DbConfig } from './interfaces';
export async function prepareEnv() {
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
const database_url = await promisify(fs.readFile)(
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
'utf-8',
);
process.env.NC_DB = jdbcToXcUrl(database_url);
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
process.env.NC_DB = jdbcToXcUrl(
process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
);
}
}
export function getToolDir() {
return process.env.NC_TOOL_DIR || process.cwd();
}
export function jdbcToXcConfig(url: string): DbConfig {
// drop the jdbc prefix
url.replace(/^jdbc:/, '');
const config = parseDbUrl(url);
const parsedConfig: Connection = {};
for (const [key, value] of Object.entries(config)) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedConfig[fnd.parameter] = value;
} else {
parsedConfig[key] = value;
}
}
if (!parsedConfig?.port) {
parsedConfig.port =
defaultClientPortMapping[
driverClientMapping[parsedConfig.driver] || parsedConfig.driver
];
}
const { driver, ...connectionConfig } = parsedConfig;
const client = driverClientMapping[driver] || driver;
if (
client === 'pg' &&
!connectionConfig?.ssl &&
!avoidSSL.includes(connectionConfig.host)
) {
connectionConfig.ssl = true;
}
return {
client: client,
connection: {
...connectionConfig,
},
} as DbConfig;
}
export function jdbcToXcUrl(url: string): string {
// drop the jdbc prefix
url.replace(/^jdbc:/, '');
const config = parseDbUrl(url);
const parsedConfig: Connection = {};
for (const [key, value] of Object.entries(config)) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedConfig[fnd.parameter] = value;
} else {
parsedConfig[key] = value;
}
}
if (!parsedConfig?.port) {
parsedConfig.port =
defaultClientPortMapping[
driverClientMapping[parsedConfig.driver] || parsedConfig.driver
];
}
const { driver, host, port, database, user, password, ...extra } =
parsedConfig;
const extraParams = [];
for (const [key, value] of Object.entries(extra)) {
extraParams.push(`${key}=${value}`);
}
const res = `${driverClientMapping[driver] || driver}://${host}${
port ? `:${port}` : ''
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
database ? `d=${database}&` : ''
}${extraParams.join('&')}`;
return res;
}
export function xcUrlToDbConfig(
urlString: string,
key = '',
type?: string,
): DbConfig {
const url = new URL(urlString);
let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) {
dbConfig = {
client: 'sqlite3',
connection: {
client: 'sqlite3',
connection: {
filename:
url.searchParams.get('d') || url.searchParams.get('database'),
},
database: url.searchParams.get('d') || url.searchParams.get('database'),
},
} as any;
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', '') as DriverClient,
connection: {
...parsedQuery,
host: url.hostname,
port: +url.port,
},
acquireConnectionTimeout: 600000,
};
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
if (
url.searchParams.get('keyFilePath') &&
url.searchParams.get('certFilePath') &&
url.searchParams.get('caFilePath')
) {
dbConfig.connection.ssl = {
keyFilePath: url.searchParams.get('keyFilePath'),
certFilePath: url.searchParams.get('certFilePath'),
caFilePath: url.searchParams.get('caFilePath'),
};
}
}
/* TODO check if this is needed
if (config && !config.title) {
config.title =
url.searchParams.get('t') ||
url.searchParams.get('title') ||
this.generateRandomTitle();
}
*/
Object.assign(dbConfig, {
meta: {
tn: 'nc_evolutions',
allSchemas:
!!url.searchParams.get('allSchemas') ||
!(url.searchParams.get('d') || url.searchParams.get('database')),
api: {
prefix: url.searchParams.get('apiPrefix') || '',
swagger: true,
type:
type ||
((url.searchParams.get('api') || url.searchParams.get('a')) as any) ||
'rest',
},
dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
metaTables: 'db',
migrations: {
disabled: false,
name: 'nc_evolutions',
},
},
});
return dbConfig;
}
export async function metaUrlToDbConfig(urlString): Promise<DbConfig> {
const url = new URL(urlString);
let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) {
const db = url.searchParams.get('d') || url.searchParams.get('database');
dbConfig = {
client: DriverClient.SQLITE,
connection: {
filename: db,
},
...(db === ':memory:'
? {
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
}
: {}),
};
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', '') as DriverClient,
connection: {
...defaultConnectionConfig,
...parsedQuery,
host: url.hostname,
port: +url.port,
},
acquireConnectionTimeout: 600000,
...defaultConnectionOptions,
...(url.searchParams.has('search_path')
? {
searchPath: url.searchParams.get('search_path').split(','),
}
: {}),
};
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
}
url.searchParams.forEach((_value, key) => {
let value: any = _value;
if (value === 'true') {
value = true;
} else if (value === 'false') {
value = false;
} else if (/^\d+$/.test(value)) {
value = +value;
}
// todo: implement config read from JSON file or JSON env val read
if (
!['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes(
key,
)
) {
key.split('.').reduce((obj, k, i, arr) => {
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
}, dbConfig);
}
});
if (
dbConfig?.connection?.ssl &&
typeof dbConfig?.connection?.ssl === 'object'
) {
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
dbConfig.connection.ssl.ca = (
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
).toString();
delete dbConfig.connection.ssl.caFilePath;
}
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
dbConfig.connection.ssl.key = (
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
).toString();
delete dbConfig.connection.ssl.keyFilePath;
}
if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) {
dbConfig.connection.ssl.cert = (
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
).toString();
delete dbConfig.connection.ssl.certFilePath;
}
}
return dbConfig;
}

4
packages/nocodb/src/utils/nc-config/index.ts

@ -0,0 +1,4 @@
export * from './helpers';
export * from './interfaces';
export * from './constants';
export * from './NcConfig';

39
packages/nocodb/src/utils/nc-config/interfaces.ts

@ -0,0 +1,39 @@
import { DriverClient } from './constants';
interface Connection {
driver?: DriverClient;
host?: string;
port?: number;
database?: string;
user?: string;
password?: string;
ssl?:
| boolean
| {
ca?: string;
cert?: string;
key?: string;
caFilePath?: string;
certFilePath?: string;
keyFilePath?: string;
};
filename?: string;
}
interface DbConfig {
client: DriverClient;
connection: Connection;
acquireConnectionTimeout?: number;
useNullAsDefault?: boolean;
pool?: {
min?: number;
max?: number;
idleTimeoutMillis?: number;
};
migrations?: {
directory?: string;
tableName?: string;
};
}
export { DriverClient, Connection, DbConfig };

12
packages/nocodb/tests/unit/TestDbMngr.ts

@ -2,9 +2,9 @@ import fs from 'fs';
import process from 'process';
import { knex } from 'knex';
import SqlMgrv2 from '../../src/db/sql-mgr/v2/SqlMgrv2';
import { jdbcToXcUrl, xcUrlToDbConfig } from '../../src/utils/nc-config';
import type { Knex } from 'knex';
import type { DbConfig } from '../../src/interface/config';
import NcConfigFactory from '../../src/utils/NcConfigFactory'
export default class TestDbMngr {
public static readonly dbName = 'test_meta';
@ -75,7 +75,7 @@ export default class TestDbMngr {
private static async isDbConfigured() {
const { user, password, host, port, client } = TestDbMngr.connection;
const config = NcConfigFactory.urlToDbConfig(
const config = xcUrlToDbConfig(
`${client}://${user}:${password}@${host}:${port}`,
);
config.connection = {
@ -84,7 +84,7 @@ export default class TestDbMngr {
host,
port,
};
const result = await TestDbMngr.testConnection(config);
const result = await TestDbMngr.testConnection(config as any);
return result.code !== -1;
}
static async connectDb() {
@ -95,9 +95,9 @@ export default class TestDbMngr {
] = `${client}://${user}:${password}@${host}:${port}/${TestDbMngr.dbName}`;
}
TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig(
NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`]),
);
TestDbMngr.dbConfig = xcUrlToDbConfig(
jdbcToXcUrl(process.env[`DATABASE_URL`]),
) as any;
this.dbConfig.meta = {
tn: 'nc_evolutions',
dbAlias: 'db',

51
packages/nocodb/tests/unit/model/tests/baseModelSql.test.ts

@ -1,20 +1,20 @@
import 'mocha';
import { BaseModelSqlv2 } from '../../../../src/db/BaseModelSqlv2'
import NcConnectionMgrv2 from '../../../../src/utils/common/NcConnectionMgrv2'
import { expect } from 'chai';
import { BaseModelSqlv2 } from '../../../../src/db/BaseModelSqlv2';
import NcConnectionMgrv2 from '../../../../src/utils/common/NcConnectionMgrv2';
import init from '../../init';
import { createProject } from '../../factory/project';
import { createTable } from '../../factory/table';
import Base from '../../../../src/models/Base';
import Model from '../../../../src/models/Model';
import Project from '../../../../src/models/Project';
import View from '../../../../src/models/View';
import { createRow, generateDefaultRowAttributes } from '../../factory/row';
import Audit from '../../../../src/models/Audit';
import { expect } from 'chai';
import Filter from '../../../../src/models/Filter';
import { createLtarColumn } from '../../factory/column';
import LinkToAnotherRecordColumn from '../../../../src/models/LinkToAnotherRecordColumn';
import { isPg, isSqlite } from '../../init/db';
import type View from '../../../../src/models/View';
import type Project from '../../../../src/models/Project';
import type Model from '../../../../src/models/Model';
import type LinkToAnotherRecordColumn from '../../../../src/models/LinkToAnotherRecordColumn';
function baseModelSqlTests() {
let context;
@ -44,11 +44,11 @@ function baseModelSqlTests() {
};
const columns = await table.getColumns();
let inputData: any = generateDefaultRowAttributes({ columns });
const inputData: any = generateDefaultRowAttributes({ columns });
const response = await baseModelSql.insert(
generateDefaultRowAttributes({ columns }),
undefined,
request
request,
);
const insertedRow = (await baseModelSql.list())[0];
@ -61,6 +61,10 @@ function baseModelSqlTests() {
response.CreatedAt = new Date(response.CreatedAt).toISOString();
response.UpdatedAt = new Date(response.UpdatedAt).toISOString();
} else if (isSqlite(context)) {
// append +00:00 to the date string
inputData.CreatedAt = `${inputData.CreatedAt}+00:00`;
inputData.UpdatedAt = `${inputData.UpdatedAt}+00:00`;
}
expect(insertedRow).to.include(inputData);
@ -106,6 +110,10 @@ function baseModelSqlTests() {
if (isPg(context)) {
inputData.CreatedAt = new Date(inputData.CreatedAt).toISOString();
inputData.UpdatedAt = new Date(inputData.UpdatedAt).toISOString();
} else if (isSqlite(context)) {
// append +00:00 to the date string
inputData.CreatedAt = `${inputData.CreatedAt}+00:00`;
inputData.UpdatedAt = `${inputData.UpdatedAt}+00:00`;
}
expect(insertedRows[index]).to.include(inputData);
});
@ -145,7 +153,7 @@ function baseModelSqlTests() {
expect(updatedRow).to.include({ Id: rowId, Title: 'test' });
const rowUpdatedAudit = (await Audit.projectAuditList(project.id, {})).find(
(audit) => audit.op_sub_type === 'UPDATE'
(audit) => audit.op_sub_type === 'UPDATE',
);
expect(rowUpdatedAudit).to.include({
user: 'test@example.com',
@ -156,7 +164,8 @@ function baseModelSqlTests() {
row_id: '1',
op_type: 'DATA',
op_sub_type: 'UPDATE',
description: 'Record with ID 1 has been updated in Table Table1_Title.\nColumn "Title" got changed from "test-0" to "test"',
description:
'Record with ID 1 has been updated in Table Table1_Title.\nColumn "Title" got changed from "test-0" to "test"',
});
});
@ -178,7 +187,7 @@ function baseModelSqlTests() {
await baseModelSql.bulkUpdate(
insertedRows.map((row) => ({ ...row, Title: `new-${row['Title']}` })),
{ cookie: request }
{ cookie: request },
);
const updatedRows = await baseModelSql.list();
@ -229,7 +238,7 @@ function baseModelSqlTests() {
],
},
{ Title: 'new-1' },
{ cookie: request }
{ cookie: request },
);
const updatedRows = await baseModelSql.list();
@ -277,7 +286,7 @@ function baseModelSqlTests() {
console.log('Delete record', await Audit.projectAuditList(project.id, {}));
const rowDeletedAudit = (await Audit.projectAuditList(project.id, {})).find(
(audit) => audit.op_sub_type === 'DELETE'
(audit) => audit.op_sub_type === 'DELETE',
);
expect(rowDeletedAudit).to.include({
user: 'test@example.com',
@ -309,7 +318,7 @@ function baseModelSqlTests() {
insertedRows
.filter((row) => row['Id'] < 5)
.map((row) => ({ id: row['Id'] })),
{ cookie: request }
{ cookie: request },
);
const remainingRows = await baseModelSql.list();
@ -359,7 +368,7 @@ function baseModelSqlTests() {
}),
],
},
{ cookie: request }
{ cookie: request },
);
const remainingRows = await baseModelSql.list();
@ -414,7 +423,7 @@ function baseModelSqlTests() {
[ltarColumn.title]: [{ Id: childRow['Id'] }],
},
undefined,
request
request,
);
const childBaseModel = new BaseModelSqlv2({
@ -470,7 +479,7 @@ function baseModelSqlTests() {
await baseModelSql.insert(
generateDefaultRowAttributes({ columns }),
undefined,
request
request,
);
const insertedRow = await baseModelSql.readByPk(1);
@ -487,7 +496,7 @@ function baseModelSqlTests() {
view,
});
const updatedChildRow = await childBaseModel.readByPk(
insertedChildRow['Id']
insertedChildRow['Id'],
);
expect(updatedChildRow[childCol.column_name]).to.equal(insertedRow['Id']);
@ -538,7 +547,7 @@ function baseModelSqlTests() {
await baseModelSql.insert(
generateDefaultRowAttributes({ columns }),
undefined,
request
request,
);
const insertedRow = await baseModelSql.readByPk(1);
@ -562,7 +571,7 @@ function baseModelSqlTests() {
view,
});
const updatedChildRow = await childBaseModel.readByPk(
insertedChildRow['Id']
insertedChildRow['Id'],
);
expect(updatedChildRow[childCol.column_name]).to.be.null;

4
packages/nocodb/tests/unit/rest/tests/tableRow.test.ts

@ -878,7 +878,9 @@ function tableTest() {
}
});
it('Sorted Formula column on rollup customer table', async function () {
// rollup usage in formula is currently not supported
// work in progress
it.skip('Sorted Formula column on rollup customer table', async function () {
const rollupColumnTitle = 'Number of rentals';
const rollupColumn = await createRollupColumn(context, {
project: sakilaProject,

4498
tests/playwright/package-lock.json generated

File diff suppressed because it is too large Load Diff

3
tests/playwright/package.json

@ -11,6 +11,7 @@
"test:repeat": "TRACE=true npx playwright test --workers=4 --repeat-each=12",
"test:quick": "TRACE=true PW_QUICK_TEST=1 npx playwright test --workers=4",
"test:debug": "./startPlayWrightServer.sh; PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console npx playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 0 --workers 1 --max-failures=1",
"test:debug:intelliJ": "TRACE=true PWDEBUG=console npx playwright test --trace on -c playwright.config.ts --headed --project=chromium --retries 0 --workers 1 --max-failures=1",
"test:debug:watch": "npx nodemon -e ts -w ./ -x \"npm run test:debug\"",
"test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console npx playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1",
"ci:test": "npx playwright test --workers=2",
@ -46,7 +47,9 @@
"body-parser": "^1.20.1",
"dayjs": "^1.11.7",
"express": "^4.18.2",
"knex": "^2.4.2",
"nocodb-sdk": "file:../../packages/nocodb-sdk",
"sqlite3": "^5.1.6",
"xlsx": "^0.18.5"
}
}

9
tests/playwright/pages/Dashboard/ExpandedForm/index.ts

@ -1,6 +1,7 @@
import { expect, Locator } from '@playwright/test';
import BasePage from '../../Base';
import { DashboardPage } from '..';
import { DateTimeCellPageObject } from '../common/Cell/DateTimeCell';
export class ExpandedFormPage extends BasePage {
readonly dashboard: DashboardPage;
@ -93,6 +94,14 @@ export class ExpandedFormPage extends BasePage {
await field.locator(`[data-testid="nc-child-list-button-link-to"]`).click();
await this.dashboard.linkRecord.select(value);
break;
case 'dateTime':
await field.locator('.nc-cell').click();
// eslint-disable-next-line no-case-declarations
const dateTimeObj = new DateTimeCellPageObject(this.dashboard.grid.cell);
await dateTimeObj.selectDate({ date: value.slice(0, 10) });
await dateTimeObj.selectTime({ hour: +value.slice(11, 13), minute: +value.slice(14, 16) });
await dateTimeObj.save();
break;
}
}

25
tests/playwright/pages/Dashboard/TreeView.ts

@ -48,6 +48,24 @@ export class TreeViewPage extends BasePage {
await this.get().locator(`.nc-project-tree-tbl-${title}`).focus();
}
async openBase({ title }: { title: string }) {
const nodes = await this.get().locator(`.ant-collapse`);
// loop through nodes.count() to find the node with title
for (let i = 0; i < (await nodes.count()); i++) {
const node = nodes.nth(i);
const nodeTitle = await node.innerText();
// check if nodeTitle contains title
if (nodeTitle.includes(title)) {
// click on node
await node.waitFor({ state: 'visible' });
await node.click();
break;
}
}
await this.rootPage.waitForTimeout(2000);
}
// assumption: first view rendered is always GRID
//
async openTable({
@ -72,6 +90,8 @@ export class TreeViewPage extends BasePage {
}
}
await this.get().locator(`.nc-project-tree-tbl-${title}`).waitFor({ state: 'visible' });
if (networkResponse === true) {
await this.waitForResponse({
uiAction: () => this.get().locator(`.nc-project-tree-tbl-${title}`).click(),
@ -86,7 +106,7 @@ export class TreeViewPage extends BasePage {
}
}
async createTable({ title, skipOpeningModal }: { title: string; skipOpeningModal?: boolean }) {
async createTable({ title, skipOpeningModal, mode }: { title: string; skipOpeningModal?: boolean; mode?: string }) {
if (!skipOpeningModal) await this.get().locator('.nc-add-new-table').click();
await this.dashboard.get().locator('.nc-modal-table-create').locator('.ant-modal-body').waitFor();
@ -101,7 +121,7 @@ export class TreeViewPage extends BasePage {
});
// Tab render is slow for playwright
await this.dashboard.waitForTabRender({ title });
await this.dashboard.waitForTabRender({ title, mode });
}
async verifyTable({ title, index, exists = true }: { title: string; index?: number; exists?: boolean }) {
@ -196,7 +216,6 @@ export class TreeViewPage extends BasePage {
uiAction: () => this.rootPage.getByRole('button', { name: 'Confirm' }).click(),
httpMethodsToMatch: ['POST'],
requestUrlPathToMatch: `/api/v1/db/meta/duplicate/`,
responseJsonMatcher: json => json.name === 'duplicate-model',
});
await this.get().locator(`[data-testid="tree-view-table-${title} copy"]`).waitFor();
}

29
tests/playwright/pages/Dashboard/common/Cell/DateTimeCell.ts

@ -23,7 +23,7 @@ export class DateTimeCellPageObject extends BasePage {
}
async save() {
await this.rootPage.locator('button:has-text("Ok")').click();
await this.rootPage.locator('button:has-text("Ok"):visible').click();
}
async selectDate({
@ -36,15 +36,15 @@ export class DateTimeCellPageObject extends BasePage {
const [year, month, day] = date.split('-');
// configure year
await this.rootPage.locator('.ant-picker-year-btn').click();
await this.rootPage.locator('.ant-picker-year-btn:visible').click();
await this.rootPage.locator(`td[title="${year}"]`).click();
// configure month
await this.rootPage.locator('.ant-picker-month-btn').click();
await this.rootPage.locator('.ant-picker-month-btn:visible').click();
await this.rootPage.locator(`td[title="${year}-${month}"]`).click();
// configure day
await this.rootPage.locator(`td[title="${year}-${month}-${day}"]`).click();
await this.rootPage.locator(`td[title="${year}-${month}-${day}"]:visible`).click();
}
async selectTime({
@ -60,14 +60,20 @@ export class DateTimeCellPageObject extends BasePage {
second?: number | null;
}) {
await this.rootPage
.locator(`.ant-picker-time-panel-column:nth-child(1) > .ant-picker-time-panel-cell:nth-child(${hour + 1})`)
.locator(
`.ant-picker-time-panel-column:nth-child(1) > .ant-picker-time-panel-cell:nth-child(${hour + 1}):visible`
)
.click();
await this.rootPage
.locator(`.ant-picker-time-panel-column:nth-child(2) > .ant-picker-time-panel-cell:nth-child(${minute + 1})`)
.locator(
`.ant-picker-time-panel-column:nth-child(2) > .ant-picker-time-panel-cell:nth-child(${minute + 1}):visible`
)
.click();
if (second != null) {
await this.rootPage
.locator(`.ant-picker-time-panel-column:nth-child(3) > .ant-picker-time-panel-cell:nth-child(${second + 1})`)
.locator(
`.ant-picker-time-panel-column:nth-child(3) > .ant-picker-time-panel-cell:nth-child(${second + 1}):visible`
)
.click();
}
}
@ -75,4 +81,13 @@ export class DateTimeCellPageObject extends BasePage {
async close() {
await this.rootPage.keyboard.press('Escape');
}
async setDateTime({ index, columnHeader, dateTime }: { index: number; columnHeader: string; dateTime: string }) {
const [date, time] = dateTime.split(' ');
const [hour, minute, second] = time.split(':');
await this.open({ index, columnHeader });
await this.selectDate({ date });
await this.selectTime({ hour: +hour, minute: +minute });
await this.save();
}
}

13
tests/playwright/pages/Dashboard/common/Cell/index.ts

@ -114,6 +114,11 @@ export class CellPageObject extends BasePage {
// if text is found, return
// if text is not found, throw error
let count = 0;
if (!(this.parent instanceof SharedFormPage)) {
await this.rootPage.locator(`td[data-testid="cell-${columnHeader}-${index}"]`).waitFor({ state: 'visible' });
}
await this.get({
index,
columnHeader,
@ -360,4 +365,12 @@ export class CellPageObject extends BasePage {
await this.get({ index, columnHeader }).press((await this.isMacOs()) ? 'Meta+C' : 'Control+C');
await this.verifyToast({ message: 'Copied to clipboard' });
}
async pasteFromClipboard({ index, columnHeader }: CellProps, ...clickOptions: Parameters<Locator['click']>) {
await this.get({ index, columnHeader }).scrollIntoViewIfNeeded();
await this.get({ index, columnHeader }).click(...clickOptions);
await (await this.get({ index, columnHeader }).elementHandle()).waitForElementState('stable');
await this.get({ index, columnHeader }).press((await this.isMacOs()) ? 'Meta+V' : 'Control+V');
}
}

1132
tests/playwright/tests/db/timezone.spec.ts

File diff suppressed because it is too large Load Diff

2
tests/playwright/tests/db/viewGridShare.spec.ts

@ -83,7 +83,7 @@ test.describe('Shared view', () => {
}
const expectedRecordsByDb = isSqlite(context) || isPg(context) ? sqliteExpectedRecords : expectedRecords;
await new Promise((resolve) => setTimeout(resolve, 1000));
await new Promise(resolve => setTimeout(resolve, 1000));
// verify order of records (original sort & filter)
for (const record of expectedRecordsByDb) {
await sharedPage.grid.cell.verify(record);

48
tests/playwright/tests/utils/config.ts

@ -0,0 +1,48 @@
const knexConfig = {
pg: {
client: 'pg',
connection: {
host: 'localhost',
port: 5432,
user: 'postgres',
password: 'password',
database: 'postgres',
multipleStatements: true,
},
searchPath: ['public', 'information_schema'],
pool: { min: 0, max: 5 },
},
mysql: {
client: 'mysql2',
connection: {
host: 'localhost',
port: 3306,
user: 'root',
password: 'password',
database: 'sakila',
multipleStatements: true,
},
pool: { min: 0, max: 5 },
},
sqlite: {
client: 'sqlite3',
connection: {
filename: './mydb.sqlite3',
},
useNullAsDefault: true,
pool: { min: 0, max: 5 },
},
};
function getKnexConfig({ dbName, dbType }: { dbName: string; dbType: string }) {
const config = knexConfig[dbType];
if (dbType === 'sqlite') {
config.connection.filename = `./${dbName}.sqlite3`;
return config;
}
config.connection.database = dbName;
return config;
}
export { getKnexConfig };

12
tests/playwright/tests/utils/general.ts

@ -50,4 +50,14 @@ function getDefaultPwd() {
return 'Password123.';
}
export { getTextExcludeIconText, isSubset, getIconText, getDefaultPwd };
function getBrowserTimezoneOffset() {
// get timezone offset
const timezoneOffset = new Date().getTimezoneOffset();
const hours = Math.floor(Math.abs(timezoneOffset) / 60);
const minutes = Math.abs(timezoneOffset % 60);
const sign = timezoneOffset <= 0 ? '+' : '-';
const formattedOffset = `${sign}${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}`;
return formattedOffset;
}
export { getTextExcludeIconText, isSubset, getIconText, getDefaultPwd, getBrowserTimezoneOffset };

Loading…
Cancel
Save