chore: sync with develop

packages/nc-gui/components/cell/DateTimePicker.vue

@@ -18,13 +18,14 @@ import {
 interface Props {
   modelValue?: string | null
   isPk?: boolean
+  isUpdatedFromCopyNPaste: Record<string, boolean>
 }
 
-const { modelValue, isPk } = defineProps<Props>()
+const { modelValue, isPk, isUpdatedFromCopyNPaste } = defineProps<Props>()
 
 const emit = defineEmits(['update:modelValue'])
 
-const { isMysql } = useProject()
+const { isMssql, isXcdbBase } = useProject()
 
 const { showNull } = useGlobal()
 
@@ -44,6 +45,8 @@ const dateTimeFormat = $computed(() => {
   return `${dateFormat} ${timeFormat}`
 })
 
+let localModelValue = modelValue ? dayjs(modelValue).utc().local() : undefined
+
 let localState = $computed({
   get() {
     if (!modelValue) {
@@ -55,7 +58,45 @@ let localState = $computed({
       return undefined
     }
 
-    return /^\d+$/.test(modelValue) ? dayjs(+modelValue) : dayjs(modelValue)
+    const isXcDB = isXcdbBase(column.value.base_id)
+
+    // cater copy and paste
+    // when copying a datetime cell, the copied value would be local time
+    // when pasting a datetime cell, UTC (xcdb) will be saved in DB
+    // we convert back to local time
+    if (column.value.title! in (isUpdatedFromCopyNPaste ?? {})) {
+      localModelValue = dayjs(modelValue).utc().local()
+      return localModelValue
+    }
+
+    // ext db
+    if (!isXcDB) {
+      return /^\d+$/.test(modelValue) ? dayjs(+modelValue) : dayjs(modelValue)
+    }
+
+    if (isMssql(column.value.base_id)) {
+      // e.g. 2023-04-29T11:41:53.000Z
+      return dayjs(modelValue)
+    }
+
+    // if cdf is defined, that means the value is auto-generated
+    // hence, show the local time
+    if (column?.value?.cdf) {
+      return dayjs(modelValue).utc().local()
+    }
+
+    // if localModelValue is defined, show localModelValue instead
+    // localModelValue is set in setter below
+    if (localModelValue) {
+      const res = localModelValue
+      // resetting localModelValue here
+      // e.g. save in expanded form -> render the correct modelValue
+      localModelValue = undefined
+      return res
+    }
+
+    // empty cell - use modelValue in local time
+    return dayjs(modelValue).utc().local()
   },
   set(val?: dayjs.Dayjs) {
     if (!val) {
@@ -64,7 +105,10 @@ let localState = $computed({
     }
 
     if (val.isValid()) {
-      emit('update:modelValue', val?.format(isMysql(column.value.base_id) ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ'))
+      // setting localModelValue to cater NOW function in date picker
+      localModelValue = dayjs(val)
+      // send the payload in UTC format
+      emit('update:modelValue', dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ'))
     }
   },
 })

packages/nc-gui/components/dashboard/TreeView.vue

@@ -399,8 +399,8 @@ const duplicateTable = async (table: TableType) => {
   const { close } = useDialog(resolveComponent('DlgTableDuplicate'), {
     'modelValue': isOpen,
     'table': table,
-    'onOk': async (jobData: { name: string; id: string }) => {
+    'onOk': async (jobData: { id: string }) => {
-      $jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string, data?: any) => {
+      $jobs.subscribe({ id: jobData.id }, undefined, async (status: string, data?: any) => {
         if (status === JobStatus.COMPLETED) {
           await loadTables()
           const newTable = tables.value.find((el) => el.id === data?.result?.id)

packages/nc-gui/components/dlg/ProjectDuplicate.vue

@@ -34,7 +34,12 @@ const isLoading = ref(false)
 const _duplicate = async () => {
   isLoading.value = true
   try {
-    const jobData = await api.project.duplicate(props.project.id as string, optionsToExclude.value)
+    const jobData = await api.project.duplicate(props.project.id as string, { 
+      options: optionsToExclude.value,
+      project: {
+        meta: props.project.meta,
+      },
+    })
     props.onOk(jobData as any)
   } catch (e: any) {
     message.error(await extractSdkResponseErrorMsg(e))

packages/nc-gui/components/dlg/TableDuplicate.vue

@@ -34,7 +34,7 @@ const isLoading = ref(false)
 const _duplicate = async () => {
   isLoading.value = true
   try {
-    const jobData = await api.dbTable.duplicate(props.table.project_id!, props.table.id!, optionsToExclude.value)
+    const jobData = await api.dbTable.duplicate(props.table.project_id!, props.table.id!, { options: optionsToExclude.value })
     props.onOk(jobData as any)
   } catch (e: any) {
     message.error(await extractSdkResponseErrorMsg(e))

packages/nc-gui/components/smartsheet/Cell.vue

@@ -209,7 +209,12 @@ onUnmounted(() => {
         <LazyCellMultiSelect v-else-if="isMultiSelect(column)" v-model="vModel" :row-index="props.rowIndex" />
         <LazyCellDatePicker v-else-if="isDate(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
         <LazyCellYearPicker v-else-if="isYear(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
-        <LazyCellDateTimePicker v-else-if="isDateTime(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
+        <LazyCellDateTimePicker
+          v-else-if="isDateTime(column, abstractType)"
+          v-model="vModel"
+          :is-pk="isPrimaryKey(column)"
+          :is-updated-from-copy-n-paste="currentRow.rowMeta.isUpdatedFromCopyNPaste"
+        />
         <LazyCellTimePicker v-else-if="isTime(column, abstractType)" v-model="vModel" :is-pk="isPrimaryKey(column)" />
         <LazyCellRating v-else-if="isRating(column)" v-model="vModel" />
         <LazyCellDuration v-else-if="isDuration(column)" v-model="vModel" />

packages/nc-gui/components/smartsheet/Grid.vue

@@ -316,6 +316,12 @@ const {
       return
     }
 
+    // See DateTimePicker.vue for details
+    data.value[ctx.row].rowMeta.isUpdatedFromCopyNPaste = {
+      ...data.value[ctx.row].rowMeta.isUpdatedFromCopyNPaste,
+      [ctx.updatedColumnTitle || columnObj.title]: true,
+    }
+
     // update/save cell value
     await updateOrSaveRow(rowObj, ctx.updatedColumnTitle || columnObj.title)
   },

packages/nc-gui/components/virtual-cell/Formula.vue

@@ -1,7 +1,7 @@
 <script lang="ts" setup>
 import type { ColumnType } from 'nocodb-sdk'
 import type { Ref } from 'vue'
-import { CellValueInj, ColumnInj, computed, handleTZ, inject, replaceUrlsWithLink, useProject } from '#imports'
+import { CellValueInj, ColumnInj, computed, handleTZ, inject, renderValue, replaceUrlsWithLink, useProject } from '#imports'
 
 // todo: column type doesn't have required property `error` - throws in typecheck
 const column = inject(ColumnInj) as Ref<ColumnType & { colOptions: { error: any } }>
@@ -10,7 +10,9 @@ const cellValue = inject(CellValueInj)
 
 const { isPg } = useProject()
 
-const result = computed(() => (isPg(column.value.base_id) ? handleTZ(cellValue?.value) : cellValue?.value))
+const result = computed(() =>
+  isPg(column.value.base_id) ? renderValue(handleTZ(cellValue?.value)) : renderValue(cellValue?.value),
+)
 
 const urls = computed(() => replaceUrlsWithLink(result.value))
 

packages/nc-gui/components/virtual-cell/components/ItemChip.vue

@@ -9,9 +9,9 @@ import {
   inject,
   isAttachment,
   ref,
+  renderValue,
   useExpandedFormDetached,
   useLTARStoreOrThrow,
-  useUIPermission,
 } from '#imports'
 
 interface Props {

packages/nc-gui/components/virtual-cell/components/ListChildItems.vue

@@ -13,10 +13,10 @@ import {
   iconMap,
   inject,
   ref,
+  renderValue,
   useLTARStoreOrThrow,
   useSmartsheetRowStoreOrThrow,
   useVModel,
-  watch,
 } from '#imports'
 
 const props = defineProps<{ modelValue?: boolean; cellValue: any; column: any }>()

packages/nc-gui/components/virtual-cell/components/ListItems.vue

@@ -12,11 +12,11 @@ import {
   inject,
   isDrawerExist,
   ref,
+  renderValue,
   useLTARStoreOrThrow,
   useSelectedCellKeyupListener,
   useSmartsheetRowStoreOrThrow,
   useVModel,
-  watch,
 } from '#imports'
 
 const props = defineProps<{ modelValue: boolean; column: any }>()
@@ -213,7 +213,7 @@ watch(vModel, (nextVal) => {
 
         <component :is="iconMap.reload" class="cursor-pointer text-gray-500 nc-reload" @click="loadChildrenExcludedList" />
 
-        <!--        Add new record -->
+        <!-- Add new record -->
         <a-button v-if="!isPublic" type="primary" size="small" @click="expandedFormDlg = true">
           {{ $t('activity.addNewRecord') }}
         </a-button>

packages/nc-gui/composables/useMultiSelect/convertCellData.ts

@@ -7,6 +7,7 @@ import { parseProp } from '#imports'
 export default function convertCellData(
   args: { from: UITypes; to: UITypes; value: any; column: ColumnType; appInfo: AppInfo },
   isMysql = false,
+  isXcdbBase = false,
 ) {
   const { from, to, value } = args
   if (from === to && ![UITypes.Attachment, UITypes.Date, UITypes.DateTime, UITypes.Time, UITypes.Year].includes(to)) {
@@ -42,7 +43,7 @@ export default function convertCellData(
       if (!parsedDateTime.isValid()) {
         throw new Error('Not a valid datetime value')
       }
-      return parsedDateTime.format(dateFormat)
+      return parsedDateTime.utc().format('YYYY-MM-DD HH:mm:ssZ')
     }
     case UITypes.Time: {
       let parsedTime = dayjs(value)

packages/nc-gui/composables/useMultiSelect/index.ts

@@ -1,3 +1,4 @@
+import dayjs from 'dayjs'
 import type { MaybeRef } from '@vueuse/core'
 import type { ColumnType, LinkToAnotherRecordType, TableType } from 'nocodb-sdk'
 import { RelationTypes, UITypes, isVirtualCol } from 'nocodb-sdk'
@@ -7,6 +8,7 @@ import convertCellData from './convertCellData'
 import type { Nullable, Row } from '~/lib'
 import {
   copyTable,
+  dateFormats,
   extractPkFromRow,
   extractSdkResponseErrorMsg,
   isMac,
@@ -14,6 +16,7 @@ import {
   message,
   reactive,
   ref,
+  timeFormats,
   unref,
   useCopy,
   useEventListener,
@@ -52,7 +55,7 @@ export function useMultiSelect(
 
   const { appInfo } = useGlobal()
 
-  const { isMysql } = useProject()
+  const { isMysql, isXcdbBase } = useProject()
 
   let clipboardContext = $ref<{ value: any; uidt: UITypes } | null>(null)
 
@@ -79,6 +82,20 @@ export function useMultiSelect(
     activeCell.col = col
   }
 
+  function constructDateTimeFormat(column: ColumnType) {
+    const dateFormat = constructDateFormat(column)
+    const timeFormat = constructTimeFormat(column)
+    return `${dateFormat} ${timeFormat}`
+  }
+
+  function constructDateFormat(column: ColumnType) {
+    return parseProp(column?.meta)?.date_format ?? dateFormats[0]
+  }
+
+  function constructTimeFormat(column: ColumnType) {
+    return parseProp(column?.meta)?.time_format ?? timeFormats[0]
+  }
+
   async function copyValue(ctx?: Cell) {
     try {
       if (selectedRange.start !== null && selectedRange.end !== null && !selectedRange.isSingleCell()) {
@@ -106,6 +123,43 @@ export function useMultiSelect(
           if (typeof textToCopy === 'object') {
             textToCopy = JSON.stringify(textToCopy)
           }
+
+          if (columnObj.uidt === UITypes.Formula) {
+            textToCopy = textToCopy.replace(/\b(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2})\b/g, (d: string) => {
+              // TODO(timezone): retrieve the format from the corresponding column meta
+              // assume hh:mm at this moment
+              return dayjs(d).utc().local().format('YYYY-MM-DD HH:mm')
+            })
+          }
+
+          if (columnObj.uidt === UITypes.DateTime || columnObj.uidt === UITypes.Time) {
+            // remove `"`
+            // e.g. "2023-05-12T08:03:53.000Z" -> 2023-05-12T08:03:53.000Z
+            textToCopy = textToCopy.replace(/["']/g, '')
+
+            const isMySQL = isMysql(columnObj.base_id)
+
+            let d = dayjs(textToCopy)
+
+            if (!d.isValid()) {
+              // insert a datetime value, copy the value without refreshing
+              // e.g. textToCopy = 2023-05-12T03:49:25.000Z
+              // feed custom parse format
+              d = dayjs(textToCopy, isMySQL ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ')
+            }
+
+            // users can change the datetime format in UI
+            // `textToCopy` would be always in YYYY-MM-DD HH:mm:ss(Z / +xx:yy) format
+            // therefore, here we reformat to the correct datetime format based on the meta
+            textToCopy = d.format(
+              columnObj.uidt === UITypes.DateTime ? constructDateTimeFormat(columnObj) : constructTimeFormat(columnObj),
+            )
+
+            if (columnObj.uidt === UITypes.DateTime && !dayjs(textToCopy).isValid()) {
+              throw new Error('Invalid DateTime')
+            }
+          }
+
           await copy(textToCopy)
           message.success(t('msg.info.copiedToClipboard'))
         }
@@ -305,6 +359,7 @@ export function useMultiSelect(
                         appInfo: unref(appInfo),
                       },
                       isMysql(meta.value?.base_id),
+                      isXcdbBase(meta.value?.base_id),
                     )
                     e.preventDefault()
 
@@ -339,6 +394,7 @@ export function useMultiSelect(
                         appInfo: unref(appInfo),
                       },
                       isMysql(meta.value?.base_id),
+                      isXcdbBase(meta.value?.base_id),
                     )
                     e.preventDefault()
                     syncCellData?.(activeCell)

packages/nc-gui/lang/fr.json

@@ -580,7 +580,7 @@
       "afterEnablePwd": "L’accès est restreint par un mot de passe",
       "privateLink": "Cette vue est partagée avec un lien privé",
       "privateLinkAdditionalInfo": "Les personnes ayant le lien privé peuvent voir uniquement les cellules visibles de cette vue",
-      "afterFormSubmitted": "Après que le formulaire ait été soumis",
+      "afterFormSubmitted": "Après que le formulaire a été soumis",
       "apiOptions": "Accéder au projet via",
       "submitAnotherForm": "Afficher le bouton \"Soumettre un autre formulaire\"",
       "showBlankForm": "Montrer un formulaire vierge après 5 secondes",

packages/nc-gui/lang/ru.json

@@ -385,10 +385,10 @@
     "nextRecord": "Следующая запись",
     "previousRecord": "Предыдущая запись",
     "copyApiURL": "Скопируйте URL API",
-    "createTable": "Create New Table",
+    "createTable": "Создать новую таблицу",
     "refreshTable": "Обновление таблицы",
-    "renameTable": "Rename Table",
+    "renameTable": "Переименовать таблицу",
-    "deleteTable": "Delete Table",
+    "deleteTable": "Удалить таблицу",
     "addField": "Добавить новое поле в эту таблицу",
     "setDisplay": "Установить как значение отображения",
     "addRow": "Добавить новую строку",

packages/nc-gui/lang/zh-Hans.json

@@ -68,7 +68,7 @@
     "questions": "问题",
     "reachOut": "联系我们",
     "betaNote": "此功能仍在测试中。",
-    "moreInfo": "这里可以找到更多信息",
+    "moreInfo": "点击此处了解更多信息。",
     "logs": "日志",
     "groupingField": "分组字段",
     "insertAfter": "在右侧插入列",
@@ -385,10 +385,10 @@
     "nextRecord": "下一条记录",
     "previousRecord": "上一条纪录",
     "copyApiURL": "复制 API 链接",
-    "createTable": "Create New Table",
+    "createTable": "创建新的表格",
     "refreshTable": "刷新表格",
-    "renameTable": "Rename Table",
+    "renameTable": "重命名表格",
-    "deleteTable": "Delete Table",
+    "deleteTable": "删除表格",
     "addField": "添加新字段",
     "setDisplay": "设置为显示值",
     "addRow": "添加新行",

packages/nc-gui/lib/types.ts

@@ -60,6 +60,8 @@ export interface Row {
     commentCount?: number
     changed?: boolean
     saving?: boolean
+    // use in datetime picker component
+    isUpdatedFromCopyNPaste?: Record<string, boolean>
   }
 }
 

packages/nc-gui/nuxt-shim.d.ts

@@ -18,7 +18,6 @@ declare module '#app/nuxt' {
         job:
           | {
               id: string
-              name: string
             }
           | any,
         subscribedCb?: () => void,

packages/nc-gui/pages/index/index/index.vue

@@ -90,10 +90,10 @@ const duplicateProject = (project: ProjectType) => {
   const { close } = useDialog(resolveComponent('DlgProjectDuplicate'), {
     'modelValue': isOpen,
     'project': project,
-    'onOk': async (jobData: { name: string; id: string }) => {
+    'onOk': async (jobData: { id: string }) => {
       await loadProjects()
 
-      $jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string) => {
+      $jobs.subscribe({ id: jobData.id }, undefined, async (status: string) => {
         if (status === JobStatus.COMPLETED) {
           await loadProjects()
         } else if (status === JobStatus.FAILED) {

packages/nc-gui/plugins/a.dayjs.ts

@@ -4,6 +4,7 @@ import customParseFormat from 'dayjs/plugin/customParseFormat.js'
 import duration from 'dayjs/plugin/duration.js'
 import utc from 'dayjs/plugin/utc.js'
 import weekday from 'dayjs/plugin/weekday.js'
+import timezone from 'dayjs/plugin/timezone.js'
 import { defineNuxtPlugin } from '#imports'
 
 export default defineNuxtPlugin(() => {
@@ -12,4 +13,5 @@ export default defineNuxtPlugin(() => {
   extend(customParseFormat)
   extend(duration)
   extend(weekday)
+  extend(timezone)
 })

packages/nc-gui/plugins/jobs.ts

@@ -29,22 +29,22 @@ export default defineNuxtPlugin(async (nuxtApp) => {
     await init(nuxtApp.$state.token.value)
   }
 
-  const send = (name: string, data: any) => {
+  const send = (evt: string, data: any) => {
     if (socket) {
       const _id = messageIndex++
-      socket.emit(name, { _id, data })
+      socket.emit(evt, { _id, data })
       return _id
     }
   }
 
   const jobs = {
     subscribe(
-      job: { id: string; name: string } | any,
+      job: { id: string } | any,
       subscribedCb?: () => void,
       statusCb?: (status: JobStatus, data?: any) => void,
       logCb?: (data: { message: string }) => void,
     ) {
-      const logFn = (data: { id: string; name: string; data: { message: string } }) => {
+      const logFn = (data: { id: string; data: { message: string } }) => {
         if (data.id === job.id) {
           if (logCb) logCb(data.data)
         }
@@ -61,11 +61,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
 
       const _id = send('subscribe', job)
 
-      const subscribeFn = (data: { _id: number; name: string; id: string }) => {
+      const subscribeFn = (data: { _id: number; id: string }) => {
         if (data._id === _id) {
-          if (data.id !== job.id || data.name !== job.name) {
+          if (data.id !== job.id) {
             job.id = data.id
-            job.name = data.name
           }
           if (subscribedCb) subscribedCb()
           socket?.on('log', logFn)
@@ -75,10 +74,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
       }
       socket?.on('subscribed', subscribeFn)
     },
-    getStatus(name: string, id: string): Promise<string> {
+    getStatus(id: string): Promise<string> {
       return new Promise((resolve) => {
         if (socket) {
-          const _id = send('status', { name, id })
+          const _id = send('status', { id })
           const tempFn = (data: any) => {
             if (data._id === _id) {
               resolve(data.status)

packages/nc-gui/store/project.ts

@@ -82,6 +82,10 @@ export const useProject = defineStore('projectStore', () => {
     return ['mysql', ClientType.MYSQL].includes(getBaseType(baseId))
   }
 
+  function isSqlite(baseId?: string) {
+    return getBaseType(baseId) === ClientType.SQLITE
+  }
+
   function isMssql(baseId?: string) {
     return getBaseType(baseId) === 'mssql'
   }
@@ -91,7 +95,7 @@ export const useProject = defineStore('projectStore', () => {
   }
 
   function isXcdbBase(baseId?: string) {
-    return bases.value.find((base) => base.id === baseId)?.is_meta
+    return (bases.value.find((base) => base.id === baseId)?.is_meta as boolean) || false
   }
 
   const isSharedBase = computed(() => projectType === 'base')
@@ -209,6 +213,7 @@ export const useProject = defineStore('projectStore', () => {
     isMysql,
     isMssql,
     isPg,
+    isSqlite,
     sqlUis,
     isSharedBase,
     loadProjectMetaInfo,

packages/nc-gui/utils/cell.ts

@@ -1,5 +1,6 @@
 import type { ColumnType } from 'nocodb-sdk'
 import { UITypes } from 'nocodb-sdk'
+import dayjs from 'dayjs'
 
 export const dataTypeLow = (column: ColumnType) => column.dt?.toLowerCase()
 export const isBoolean = (column: ColumnType, abstractType?: any) =>
@@ -53,3 +54,25 @@ export const isManualSaved = (column: ColumnType) => [UITypes.Currency].includes
 export const isPrimary = (column: ColumnType) => !!column.pv
 
 export const isPrimaryKey = (column: ColumnType) => !!column.pk
+
+// used for LTAR and Formula
+export const renderValue = (result?: any) => {
+  if (!result || typeof result !== 'string') {
+    return result
+  }
+
+  // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
+  // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
+  result = result.replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, (d: string) => {
+    return dayjs(d).isValid() ? dayjs(d).format('YYYY-MM-DD HH:mm:ssZ') : d
+  })
+
+  // convert all date time values to local time
+  // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
+  // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
+  return result.replace(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g, (d: string) => {
+    // TODO(timezone): retrieve the format from the corresponding column meta
+    // assume HH:mm at this moment
+    return dayjs(d).isValid() ? dayjs(d).format('YYYY-MM-DD HH:mm') : d
+  })
+}

packages/nc-gui/utils/dateTimeUtils.ts

@@ -1,7 +1,13 @@
 import dayjs from 'dayjs'
 
 export const timeAgo = (date: any) => {
-  return dayjs.utc(date).fromNow()
+  if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(date)) {
+    // if there is no timezone info, consider as UTC
+    // e.g. 2023-01-01 08:00:00 (MySQL)
+    date += '+00:00'
+  }
+  // show in local time
+  return dayjs(date).fromNow()
 }
 
 export const dateFormats = [

packages/nc-gui/utils/index.ts

@@ -23,3 +23,4 @@ export * from './browserUtils'
 export * from './geoDataUtils'
 export * from './mimeTypeUtils'
 export * from './parseUtils'
+export * from './cell'

packages/nocodb-sdk/src/lib/Api.ts

@@ -4032,9 +4032,12 @@ export class Api<
     baseDuplicate: (
       projectId: IdType,
       data: {
-        excludeData?: boolean;
+        options?: {
-        excludeViews?: boolean;
+          excludeData?: boolean;
-        excludeHooks?: boolean;
+          excludeViews?: boolean;
+          excludeHooks?: boolean;
+        };
+        project?: object;
       },
       baseId?: IdType,
       params: RequestParams = {}
@@ -4078,9 +4081,12 @@ export class Api<
     duplicate: (
       projectId: IdType,
       data: {
-        excludeData?: boolean;
+        options?: {
-        excludeViews?: boolean;
+          excludeData?: boolean;
-        excludeHooks?: boolean;
+          excludeViews?: boolean;
+          excludeHooks?: boolean;
+        };
+        project?: object;
       },
       params: RequestParams = {}
     ) =>
@@ -5118,8 +5124,11 @@ export class Api<
       projectId: IdType,
       tableId: IdType,
       data: {
-        excludeData?: boolean;
+        options?: {
-        excludeViews?: boolean;
+          excludeData?: boolean;
+          excludeViews?: boolean;
+          excludeHooks?: boolean;
+        };
       },
       params: RequestParams = {}
     ) =>

packages/nocodb/src/app.module.ts

@@ -1,6 +1,5 @@
 import { Module, RequestMethod } from '@nestjs/common';
 import { APP_FILTER } from '@nestjs/core';
-import { BullModule } from '@nestjs/bull';
 import { EventEmitterModule as NestJsEventEmitter } from '@nestjs/event-emitter';
 import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter';
 import { GlobalMiddleware } from './middlewares/global/global.middleware';
@@ -17,7 +16,6 @@ import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.str
 import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy';
 import { MetasModule } from './modules/metas/metas.module';
 import { JobsModule } from './modules/jobs/jobs.module';
-import { AppInitService } from './services/app-init.service';
 import type { MiddlewareConsumer } from '@nestjs/common';
 
 @Module({
@@ -30,13 +28,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
     EventEmitterModule,
     JobsModule,
     NestJsEventEmitter.forRoot(),
-    ...(process.env['NC_REDIS_URL']
-      ? [
-          BullModule.forRoot({
-            redis: process.env.NC_REDIS_URL,
-          }),
-        ]
-      : []),
   ],
   controllers: [],
   providers: [
@@ -49,7 +40,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
     AuthTokenStrategy,
     BaseViewStrategy,
     HookHandlerService,
-    AppInitService,
   ],
 })
 export class AppModule {

packages/nocodb/src/cache/RedisCacheMgr.ts

@@ -12,8 +12,12 @@ export default class RedisCacheMgr extends CacheMgr {
   constructor(config: any) {
     super();
     this.client = new Redis(config);
-    // flush the existing db with selected key (Default: 0)
+
-    this.client.flushdb();
+    // avoid flushing db in worker container
+    if (process.env.NC_WORKER_CONTAINER !== 'true') {
+      // flush the existing db with selected key (Default: 0)
+      this.client.flushdb();
+    }
 
     // TODO(cache): fetch orgs once it's implemented
     const orgs = 'noco';

packages/nocodb/src/connection/connection.spec.ts

@@ -1,19 +0,0 @@
-import { Test } from '@nestjs/testing';
-import { Connection } from './knex';
-import type { TestingModule } from '@nestjs/testing';
-
-describe('Knex', () => {
-  let provider: Connection;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [Connection],
-    }).compile();
-
-    provider = module.get<Connection>(Connection);
-  });
-
-  it('should be defined', () => {
-    expect(provider).toBeDefined();
-  });
-});

packages/nocodb/src/connection/connection.ts

@@ -1,37 +0,0 @@
-import { Global, Injectable, Scope } from '@nestjs/common';
-
-import { XKnex } from '../db/CustomKnex';
-import NcConfigFactory from '../utils/NcConfigFactory';
-import type * as knex from 'knex';
-
-@Injectable({
-  scope: Scope.DEFAULT,
-})
-export class Connection {
-  public static knex: knex.Knex;
-  public static _config: any;
-
-  get knexInstance(): knex.Knex {
-    return Connection.knex;
-  }
-
-  get config(): knex.Knex {
-    return Connection._config;
-  }
-
-  // init metadb connection
-  static async init(): Promise<void> {
-    Connection._config = await NcConfigFactory.make();
-    if (!Connection.knex) {
-      Connection.knex = XKnex({
-        ...this._config.meta.db,
-        useNullAsDefault: true,
-      });
-    }
-  }
-
-  // init metadb connection
-  async init(): Promise<void> {
-    return await Connection.init();
-  }
-}

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -1,6 +1,9 @@
 import autoBind from 'auto-bind';
 import groupBy from 'lodash/groupBy';
 import DataLoader from 'dataloader';
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc.js';
+import timezone from 'dayjs/plugin/timezone';
 import { nocoExecute } from 'nc-help';
 import {
   AuditOperationSubTypes,
@@ -16,8 +19,16 @@ import DOMPurify from 'isomorphic-dompurify';
 import { v4 as uuidv4 } from 'uuid';
 import { NcError } from '../helpers/catchError';
 import getAst from '../helpers/getAst';
-
+import {
-import { Audit, Column, Filter, Model, Project, Sort, View } from '../models';
+  Audit,
+  Base,
+  Column,
+  Filter,
+  Model,
+  Project,
+  Sort,
+  View,
+} from '../models';
 import { sanitize, unsanitize } from '../helpers/sqlSanitize';
 import {
   COMPARISON_OPS,
@@ -46,7 +57,10 @@ import type {
   SelectOption,
 } from '../models';
 import type { Knex } from 'knex';
-import type { SortType } from 'nocodb-sdk';
+import type { BoolType, SortType } from 'nocodb-sdk';
+
+dayjs.extend(utc);
+dayjs.extend(timezone);
 
 const GROUP_COL = '__nc_group_id';
 
@@ -1607,6 +1621,67 @@ class BaseModelSqlv2 {
       if (!checkColumnRequired(column, fields, extractPkAndPv)) continue;
 
       switch (column.uidt) {
+        case UITypes.DateTime:
+          if (this.isMySQL) {
+            // MySQL stores timestamp in UTC but display in timezone
+            // To verify the timezone, run `SELECT @@global.time_zone, @@session.time_zone;`
+            // If it's SYSTEM, then the timezone is read from the configuration file
+            // if a timezone is set in a DB, the retrieved value would be converted to the corresponding timezone
+            // for example, let's say the global timezone is +08:00 in DB
+            // the value 2023-01-01 10:00:00 (UTC) would display as 2023-01-01 18:00:00 (UTC+8)
+            // our existing logic is based on UTC, during the query, we need to take the UTC value
+            // hence, we use CONVERT_TZ to convert back to UTC value
+            res[sanitize(column.title || column.column_name)] =
+              this.dbDriver.raw(
+                `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`,
+                [
+                  `${sanitize(alias || this.model.table_name)}.${
+                    column.column_name
+                  }`,
+                ],
+              );
+            break;
+          } else if (this.isPg) {
+            // if there is no timezone info,
+            // convert to database timezone,
+            // then convert to UTC
+            if (
+              column.dt !== 'timestamp with time zone' &&
+              column.dt !== 'timestamptz'
+            ) {
+              res[sanitize(column.title || column.column_name)] = this.dbDriver
+                .raw(
+                  `?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`,
+                  [
+                    `${sanitize(alias || this.model.table_name)}.${
+                      column.column_name
+                    }`,
+                  ],
+                )
+                .wrap('(', ')');
+              break;
+            }
+          } else if (this.isMssql) {
+            // if there is no timezone info,
+            // convert to database timezone,
+            // then convert to UTC
+            if (column.dt !== 'datetimeoffset') {
+              res[sanitize(column.title || column.column_name)] =
+                this.dbDriver.raw(
+                  `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
+                  [
+                    `${sanitize(alias || this.model.table_name)}.${
+                      column.column_name
+                    }`,
+                  ],
+                );
+              break;
+            }
+          }
+          res[sanitize(column.title || column.column_name)] = sanitize(
+            `${alias || this.model.table_name}.${column.column_name}`,
+          );
+          break;
         case 'LinkToAnotherRecord':
         case 'Lookup':
           break;
@@ -1734,7 +1809,11 @@ class BaseModelSqlv2 {
       await populatePk(this.model, data);
 
       // todo: filter based on view
-      const insertObj = await this.model.mapAliasToColumn(data);
+      const insertObj = await this.model.mapAliasToColumn(
+        data,
+        this.clientMeta,
+        this.dbDriver,
+      );
 
       await this.validate(insertObj);
 
@@ -1872,7 +1951,11 @@ class BaseModelSqlv2 {
 
   async updateByPk(id, data, trx?, cookie?) {
     try {
-      const updateObj = await this.model.mapAliasToColumn(data);
+      const updateObj = await this.model.mapAliasToColumn(
+        data,
+        this.clientMeta,
+        this.dbDriver,
+      );
 
       await this.validate(data);
 
@@ -1920,6 +2003,16 @@ class BaseModelSqlv2 {
     return this.getTnPath(this.model);
   }
 
+  public get clientMeta() {
+    return {
+      isSqlite: this.isSqlite,
+      isMssql: this.isMssql,
+      isPg: this.isPg,
+      isMySQL: this.isMySQL,
+      // isSnowflake: this.isSnowflake,
+    };
+  }
+
   get isSqlite() {
     return this.clientType === 'sqlite3';
   }
@@ -1948,7 +2041,11 @@ class BaseModelSqlv2 {
     // const driver = trx ? trx : await this.dbDriver.transaction();
     try {
       await populatePk(this.model, data);
-      const insertObj = await this.model.mapAliasToColumn(data);
+      const insertObj = await this.model.mapAliasToColumn(
+        data,
+        this.clientMeta,
+        this.dbDriver,
+      );
 
       let rowId = null;
       const postInsertOps = [];
@@ -2106,7 +2203,11 @@ class BaseModelSqlv2 {
         : await Promise.all(
             datas.map(async (d) => {
               await populatePk(this.model, d);
-              return this.model.mapAliasToColumn(d);
+              return this.model.mapAliasToColumn(
+                d,
+                this.clientMeta,
+                this.dbDriver,
+              );
             }),
           );
 
@@ -2170,7 +2271,11 @@ class BaseModelSqlv2 {
 
       const updateDatas = raw
         ? datas
-        : await Promise.all(datas.map((d) => this.model.mapAliasToColumn(d)));
+        : await Promise.all(
+            datas.map((d) =>
+              this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver),
+            ),
+          );
 
       const prevData = [];
       const newData = [];
@@ -2222,7 +2327,11 @@ class BaseModelSqlv2 {
   ) {
     try {
       let count = 0;
-      const updateData = await this.model.mapAliasToColumn(data);
+      const updateData = await this.model.mapAliasToColumn(
+        data,
+        this.clientMeta,
+        this.dbDriver,
+      );
       await this.validate(updateData);
       const pkValues = await this._extractPksValues(updateData);
       if (pkValues) {
@@ -2268,7 +2377,9 @@ class BaseModelSqlv2 {
     let transaction;
     try {
       const deleteIds = await Promise.all(
-        ids.map((d) => this.model.mapAliasToColumn(d)),
+        ids.map((d) =>
+          this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver),
+        ),
       );
 
       const deleted = [];
@@ -3154,16 +3265,23 @@ class BaseModelSqlv2 {
     } else {
       query = sanitize(query);
     }
-    return this.convertAttachmentType(
+
+    let data =
       this.isPg || this.isSnowflake
         ? (await this.dbDriver.raw(query))?.rows
         : query.slice(0, 6) === 'select' && !this.isMssql
         ? await this.dbDriver.from(
             this.dbDriver.raw(query).wrap('(', ') __nc_alias'),
           )
-        : await this.dbDriver.raw(query),
+        : await this.dbDriver.raw(query);
-      childTable,
+
-    );
+    // update attachment fields
+    data = this.convertAttachmentType(data, childTable);
+
+    // update date time fields
+    data = this.convertDateFormat(data, childTable);
+
+    return data;
   }
 
   private _convertAttachmentType(
@@ -3195,7 +3313,153 @@ class BaseModelSqlv2 {
             this._convertAttachmentType(attachmentColumns, d),
           );
         } else {
-          this._convertAttachmentType(attachmentColumns, data);
+          data = this._convertAttachmentType(attachmentColumns, data);
+        }
+      }
+    }
+    return data;
+  }
+
+  // TODO(timezone): retrieve the format from the corresponding column meta
+  private _convertDateFormat(
+    dateTimeColumns: Record<string, any>[],
+    d: Record<string, any>,
+  ) {
+    if (!d) return d;
+    for (const col of dateTimeColumns) {
+      if (!d[col.title]) continue;
+
+      if (col.uidt === UITypes.Formula) {
+        if (!d[col.title] || typeof d[col.title] !== 'string') {
+          continue;
+        }
+
+        // remove milliseconds
+        if (this.isMySQL) {
+          d[col.title] = d[col.title].replace(/\.000000/g, '');
+        } else if (this.isMssql) {
+          d[col.title] = d[col.title].replace(/\.0000000 \+00:00/g, '');
+        }
+
+        if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.title])) {
+          // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
+          // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
+          d[col.title] = d[col.title].replace(
+            /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g,
+            (d: string) => {
+              if (!dayjs(d).isValid()) return d;
+              if (this.isSqlite) {
+                // e.g. DATEADD formula
+                return dayjs(d).utc().format('YYYY-MM-DD HH:mm:ssZ');
+              }
+              return dayjs(d).utc(true).format('YYYY-MM-DD HH:mm:ssZ');
+            },
+          );
+          continue;
+        }
+
+        // convert all date time values to utc
+        // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
+        // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
+        d[col.title] = d[col.title].replace(
+          /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g,
+          (d: string) => {
+            if (!dayjs(d).isValid()) {
+              return d;
+            }
+
+            if (this.isSqlite) {
+              // if there is no timezone info,
+              // we assume the input is on NocoDB server timezone
+              // then we convert to UTC from server timezone
+              // example: datetime without timezone
+              // we need to display 2023-04-27 10:00:00 (in HKT)
+              // we convert d (e.g. 2023-04-27 18:00:00) to utc, i.e. 2023-04-27 02:00:00+00:00
+              // if there is timezone info,
+              // we simply convert it to UTC
+              // example: datetime with timezone
+              // e.g. 2023-04-27 10:00:00+05:30  -> 2023-04-27 04:30:00+00:00
+              return dayjs(d)
+                .tz(Intl.DateTimeFormat().resolvedOptions().timeZone)
+                .utc()
+                .format('YYYY-MM-DD HH:mm:ssZ');
+            }
+
+            // set keepLocalTime to true if timezone info is not found
+            const keepLocalTime = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/g.test(
+              d,
+            );
+
+            return dayjs(d).utc(keepLocalTime).format('YYYY-MM-DD HH:mm:ssZ');
+          },
+        );
+        continue;
+      }
+
+      let keepLocalTime = true;
+
+      if (this.isSqlite) {
+        if (!col.cdf) {
+          if (
+            d[col.title].indexOf('-') === -1 &&
+            d[col.title].indexOf('+') === -1 &&
+            d[col.title].slice(-1) !== 'Z'
+          ) {
+            // if there is no timezone info,
+            // we assume the input is on NocoDB server timezone
+            // then we convert to UTC from server timezone
+            // e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00
+            d[col.title] = dayjs(d[col.title])
+              .tz(Intl.DateTimeFormat().resolvedOptions().timeZone)
+              .utc()
+              .format('YYYY-MM-DD HH:mm:ssZ');
+            continue;
+          } else {
+            // otherwise, we convert from the given timezone to UTC
+            keepLocalTime = false;
+          }
+        }
+      }
+
+      if (
+        this.isPg &&
+        (col.dt === 'timestamp with time zone' || col.dt === 'timestamptz')
+      ) {
+        // postgres - timezone already attached to input
+        // e.g. 2023-05-11 16:16:51+08:00
+        keepLocalTime = false;
+      }
+
+      if (d[col.title] instanceof Date) {
+        // e.g. MSSQL
+        // Wed May 10 2023 17:47:46 GMT+0800 (Hong Kong Standard Time)
+        keepLocalTime = false;
+      }
+      // e.g. 01.01.2022 10:00:00+05:30 -> 2022-01-01 04:30:00+00:00
+      // e.g. 2023-05-09 11:41:49 -> 2023-05-09 11:41:49+00:00
+      d[col.title] = dayjs(d[col.title])
+        // keep the local time
+        .utc(keepLocalTime)
+        // show the timezone even for Mysql
+        .format('YYYY-MM-DD HH:mm:ssZ');
+    }
+    return d;
+  }
+
+  private convertDateFormat(data: Record<string, any>, childTable?: Model) {
+    // Show the date time in UTC format in API response
+    // e.g. 2022-01-01 04:30:00+00:00
+    if (data) {
+      const dateTimeColumns = (
+        childTable ? childTable.columns : this.model.columns
+      ).filter(
+        (c) => c.uidt === UITypes.DateTime || c.uidt === UITypes.Formula,
+      );
+      if (dateTimeColumns.length) {
+        if (Array.isArray(data)) {
+          data = data.map((d) => this._convertDateFormat(dateTimeColumns, d));
+        } else {
+          data = this._convertDateFormat(dateTimeColumns, data);
         }
       }
     }

packages/nocodb/src/db/CustomKnex.ts

@@ -1,12 +1,24 @@
 import { Knex, knex } from 'knex';
 import { SnowflakeClient } from 'nc-help';
-import { types } from 'pg';
+import pg, { types } from 'pg';
+import dayjs from 'dayjs';
 import Filter from '../models/Filter';
 import type { FilterType } from 'nocodb-sdk';
 import type { BaseModelSql } from './BaseModelSql';
 
+// For the code, check out
+// https://raw.githubusercontent.com/brianc/node-pg-types/master/lib/builtins.js
+
 // override parsing date column to Date()
 types.setTypeParser(1082, (val) => val);
+// override timestamp
+types.setTypeParser(1114, (val) => {
+  return dayjs(val).format('YYYY-MM-DD HH:mm:ss');
+});
+// override timestampz
+types.setTypeParser(1184, (val) => {
+  return dayjs(val).format('YYYY-MM-DD HH:mm:ssZ');
+});
 
 const opMappingGen = {
   eq: '=',

packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts

@@ -537,6 +537,51 @@ async function _formulaQueryBuilder(
             };
         };
         break;
+      case UITypes.DateTime:
+        if (knex.clientType().startsWith('mysql')) {
+          aliasToColumn[col.id] = async (): Promise<any> => {
+            return {
+              // convert from DB timezone to UTC
+              builder: knex.raw(
+                `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`,
+                [col.column_name],
+              ),
+            };
+          };
+        } else if (
+          knex.clientType() === 'pg' &&
+          col.dt !== 'timestamp with time zone' &&
+          col.dt !== 'timestamptz'
+        ) {
+          aliasToColumn[col.id] = async (): Promise<any> => {
+            return {
+              // convert from DB timezone to UTC
+              builder: knex
+                .raw(
+                  `?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`,
+                  [col.column_name],
+                )
+                .wrap('(', ')'),
+            };
+          };
+        } else if (
+          knex.clientType() === 'mssql' &&
+          col.dt !== 'datetimeoffset'
+        ) {
+          // convert from DB timezone to UTC
+          aliasToColumn[col.id] = async (): Promise<any> => {
+            return {
+              builder: knex.raw(
+                `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
+                [col.column_name],
+              ),
+            };
+          };
+        } else {
+          aliasToColumn[col.id] = () =>
+            Promise.resolve({ builder: col.column_name });
+        }
+        break;
       default:
         aliasToColumn[col.id] = () =>
           Promise.resolve({ builder: col.column_name });

packages/nocodb/src/db/functionMappings/mssql.ts

@@ -130,15 +130,15 @@ const mssql = {
         )}, 
         ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
           (await fn(pt.arguments[0])).builder
-        }), 'yyyy-MM-dd HH:mm')
+        }), 'yyyy-MM-dd HH:mm:ss')
       ELSE
        FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace(
          /["']/g,
          '',
        )}, 
-       ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${fn(
+       ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
-          pt.arguments[0],
+          (await fn(pt.arguments[0])).builder
-        )}), 'yyyy-MM-dd')
+        }), 'yyyy-MM-dd')
       END${colAlias}`,
       ),
     };

packages/nocodb/src/db/functionMappings/mysql.ts

@@ -62,7 +62,7 @@ const mysql2 = {
         DATE_FORMAT(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL
         ${(await fn(pt.arguments[1])).builder} ${String(
           (await fn(pt.arguments[2])).builder,
-        ).replace(/["']/g, '')}), '%Y-%m-%d %H:%i')
+        ).replace(/["']/g, '')}), '%Y-%m-%d %H:%i:%s')
       ELSE
         DATE(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL
         ${(await fn(pt.arguments[1])).builder} ${String(

packages/nocodb/src/db/functionMappings/sqlite.ts

@@ -95,9 +95,9 @@ const sqlite3 = {
       builder: knex.raw(
         `CASE
       WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN
-        STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${
+        STRFTIME('%Y-%m-%dT%H:%M:%fZ', DATETIME(DATETIME(${
           (await fn(pt.arguments[0])).builder
-        }, 'localtime'),
+        }, 'utc'),
         ${dateIN > 0 ? '+' : ''}${
           (await fn(pt.arguments[1])).builder
         } || ' ${String((await fn(pt.arguments[2])).builder).replace(
@@ -105,7 +105,7 @@ const sqlite3 = {
           '',
         )}'))
       ELSE
-        DATE(DATETIME(${(await fn(pt.arguments[0])).builder}, 'localtime'),
+        DATE(DATETIME(${(await fn(pt.arguments[0])).builder}),
         ${dateIN > 0 ? '+' : ''}${
           (await fn(pt.arguments[1])).builder
         } || ' ${String((await fn(pt.arguments[2])).builder).replace(

packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts

@@ -10,7 +10,7 @@ import Debug from '../../util/Debug';
 import Emit from '../../util/emit';
 import Result from '../../util/Result';
 import * as fileHelp from '../../util/file.help';
-import NcConfigFactory from '../../../utils/NcConfigFactory';
+import { getToolDir, NcConfig } from '../../../utils/nc-config';
 import SqlMigrator from './SqlMigrator';
 
 const evt = new Emit();
@@ -39,7 +39,7 @@ export default class KnexMigrator extends SqlMigrator {
     this.project_id = projectObj?.project_id;
     this.project = projectObj?.config;
     this.metaDb = projectObj?.metaDb;
-    this.toolDir = NcConfigFactory.getToolDir();
+    this.toolDir = getToolDir();
   }
 
   emit(data, _args?) {
@@ -312,8 +312,12 @@ export default class KnexMigrator extends SqlMigrator {
       if (exists) {
         await this._readProjectJson(projJsonFilePath);
         this.emit('Migrator for project initalised successfully');
-      } else if (NcConfigFactory.hasDbUrl()) {
+      } else if (
-        this.project = await NcConfigFactory.make();
+        Object.keys(process.env).some((envKey) =>
+          envKey.startsWith('NC_DB_URL'),
+        )
+      ) {
+        this.project = await NcConfig.createByEnv();
       } else {
         args.type = args.type || 'sqlite';
 

packages/nocodb/src/index.ts

@@ -1,6 +1,5 @@
 import Noco from './Noco';
-import NcConfigFactory from './utils/NcConfigFactory';
 
 export default Noco;
 
-export { Noco, NcConfigFactory };
+export { Noco };

packages/nocodb/src/init.ts

@@ -1,12 +1,12 @@
-import { Connection } from './connection/connection';
 import { MetaService } from './meta/meta.service';
+import { NcConfig } from './utils/nc-config';
 import Noco from './Noco';
 
 // run upgrader
 import NcUpgrader from './version-upgrader/NcUpgrader';
 
 export default async () => {
-  await Connection.init();
+  const config = await NcConfig.createByEnv();
-  Noco._ncMeta = new MetaService(new Connection());
+  Noco._ncMeta = new MetaService(config);
   await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
 };

packages/nocodb/src/main.ts

@@ -5,12 +5,22 @@ import { AppModule } from './app.module';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule);
-  app.use(express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }));
+  if (process.env.NC_WORKER_CONTAINER !== 'true') {
-  app.use(
+    app.use(
-    cors({
+      express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }),
-      exposedHeaders: 'xc-db-response',
+    );
-    }),
+    app.use(
-  );
+      cors({
-  await app.listen(8080);
+        exposedHeaders: 'xc-db-response',
+      }),
+    );
+    await app.listen(8080);
+  } else {
+    if (!process.env.NC_REDIS_URL) {
+      throw new Error('NC_REDIS_URL is required');
+    }
+    process.env.NC_DISABLE_TELE = 'true';
+    await app.init();
+  }
 }
 bootstrap();

packages/nocodb/src/meta/meta.service.ts

@@ -1,20 +1,18 @@
-import {
+import { Injectable, Optional } from '@nestjs/common';
-  Global,
+import dayjs from 'dayjs';
-  Inject,
+import utc from 'dayjs/plugin/utc';
-  Injectable,
+import timezone from 'dayjs/plugin/timezone';
-  OnApplicationBootstrap,
-  OnModuleInit,
-  Optional,
-} from '@nestjs/common';
-
 import { customAlphabet } from 'nanoid';
 import CryptoJS from 'crypto-js';
-import { Connection } from '../connection/connection';
+import { XKnex } from '../db/CustomKnex';
-import Noco from '../Noco';
+import { NcConfig } from '../utils/nc-config';
-import NocoCache from '../cache/NocoCache';
 import XcMigrationSourcev2 from './migrations/XcMigrationSourcev2';
 import XcMigrationSource from './migrations/XcMigrationSource';
 import type { Knex } from 'knex';
+import type * as knex from 'knex';
+
+dayjs.extend(utc);
+dayjs.extend(timezone);
 
 const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4);
 
@@ -187,18 +185,38 @@ const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14);
 
 @Injectable()
 export class MetaService {
-  constructor(private metaConnection: Connection, @Optional() trx = null) {
+  private _knex: knex.Knex;
+  private _config: any;
+
+  constructor(config: NcConfig, @Optional() trx = null) {
+    this._config = config;
+    this._knex = XKnex({
+      ...this._config.meta.db,
+      useNullAsDefault: true,
+    });
     this.trx = trx;
   }
 
+  get knexInstance(): knex.Knex {
+    return this._knex;
+  }
+
+  get config(): NcConfig {
+    return this._config;
+  }
+
   public get connection() {
-    return this.trx ?? this.metaConnection.knexInstance;
+    return this.trx ?? this.knexInstance;
   }
 
   get knexConnection() {
     return this.connection;
   }
 
+  public get knex(): any {
+    return this.knexConnection;
+  }
+
   public async metaGet(
     project_id: string,
     dbAlias: string,
@@ -256,8 +274,8 @@ export class MetaService {
 
     await this.knexConnection(target).insert({
       ...insertObj,
-      created_at: data?.created_at || this.knexConnection?.fn?.now(),
+      created_at: this.now(),
-      updated_at: data?.updated_at || this.knexConnection?.fn?.now(),
+      updated_at: this.now(),
     });
     return insertObj;
   }
@@ -539,9 +557,9 @@ export class MetaService {
     return this.knexConnection(target).insert({
       db_alias: dbAlias,
       project_id,
-      created_at: this.knexConnection?.fn?.now(),
-      updated_at: this.knexConnection?.fn?.now(),
       ...data,
+      created_at: this.now(),
+      updated_at: this.now(),
     });
   }
 
@@ -689,7 +707,7 @@ export class MetaService {
 
     delete data.created_at;
 
-    query.update({ ...data, updated_at: this.knexConnection?.fn?.now() });
+    query.update({ ...data, updated_at: this.now() });
     if (typeof idOrCondition !== 'object') {
       query.where('id', idOrCondition);
     } else if (idOrCondition) {
@@ -753,7 +771,7 @@ export class MetaService {
     });
 
     // todo: tobe done
-    return new MetaService(this.metaConnection, trx);
+    return new MetaService(this.config, trx);
   }
 
   async metaReset(
@@ -810,8 +828,8 @@ export class MetaService {
       // todo: check project name used or not
       await this.knexConnection('nc_projects').insert({
         ...project,
-        created_at: this.knexConnection?.fn?.now(),
+        created_at: this.now(),
-        updated_at: this.knexConnection?.fn?.now(),
+        updated_at: this.now(),
       });
 
       // todo
@@ -1022,14 +1040,23 @@ export class MetaService {
       .delete();
   }
 
-  public get knex(): any {
-    return this.knexConnection;
-  }
-
   private getNanoId() {
     return nanoid();
   }
 
+  private isMySQL(): boolean {
+    return (
+      this.connection.clientType() === 'mysql' ||
+      this.connection.clientType() === 'mysql2'
+    );
+  }
+
+  private now(): any {
+    return dayjs()
+      .utc()
+      .format(this.isMySQL() ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ');
+  }
+
   public async audit(
     project_id: string,
     dbAlias: string,

packages/nocodb/src/models/Model.ts

@@ -1,4 +1,5 @@
 import { isVirtualCol, ModelTypes, UITypes, ViewTypes } from 'nocodb-sdk';
+import dayjs from 'dayjs';
 import { BaseModelSqlv2 } from '../db/BaseModelSqlv2';
 import Noco from '../Noco';
 import { parseMetaProp } from '../utils/modelUtils';
@@ -15,6 +16,7 @@ import { sanitize } from '../helpers/sqlSanitize';
 import { extractProps } from '../helpers/extractProps';
 import Audit from './Audit';
 import View from './View';
+import Base from './Base';
 import Column from './Column';
 import type { BoolType, TableReqType, TableType } from 'nocodb-sdk';
 import type { XKnex } from '../db/CustomKnex';
@@ -458,8 +460,18 @@ export default class Model implements TableType {
     return true;
   }
 
-  async mapAliasToColumn(data) {
+  async mapAliasToColumn(
+    data,
+    clientMeta = {
+      isMySQL: false,
+      isSqlite: false,
+      isMssql: false,
+      isPg: false,
+    },
+    knex,
+  ) {
     const insertObj = {};
+    const base = await Base.get(this.base_id);
     for (const col of await this.getColumns()) {
       if (isVirtualCol(col)) continue;
       let val =
@@ -470,6 +482,57 @@ export default class Model implements TableType {
         if (col.uidt === UITypes.Attachment && typeof val !== 'string') {
           val = JSON.stringify(val);
         }
+        if (col.uidt === UITypes.DateTime && dayjs(val).isValid()) {
+          const { isMySQL, isSqlite, isMssql, isPg } = clientMeta;
+          if (
+            val.indexOf('-') < 0 &&
+            val.indexOf('+') < 0 &&
+            val.slice(-1) !== 'Z'
+          ) {
+            // if no timezone is given,
+            // then append +00:00 to make it as UTC
+            val += '+00:00';
+          }
+          if (isMySQL) {
+            // first convert the value to utc
+            // from UI
+            // e.g. 2022-01-01 20:00:00Z -> 2022-01-01 20:00:00
+            // from API
+            // e.g. 2022-01-01 20:00:00+08:00 -> 2022-01-01 12:00:00
+            // if timezone info is not found - considered as utc
+            // e.g. 2022-01-01 20:00:00 -> 2022-01-01 20:00:00
+            // if timezone info is found
+            // e.g. 2022-01-01 20:00:00Z -> 2022-01-01 20:00:00
+            // e.g. 2022-01-01 20:00:00+00:00 -> 2022-01-01 20:00:00
+            // e.g. 2022-01-01 20:00:00+08:00 -> 2022-01-01 12:00:00
+            // then we use CONVERT_TZ to convert that in the db timezone
+            val = knex.raw(`CONVERT_TZ(?, '+00:00', @@GLOBAL.time_zone)`, [
+              dayjs(val).utc().format('YYYY-MM-DD HH:mm:ss'),
+            ]);
+          } else if (isSqlite) {
+            // convert to UTC
+            // e.g. 2022-01-01T10:00:00.000Z -> 2022-01-01 04:30:00+00:00
+            val = dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ');
+          } else if (isPg) {
+            // convert to UTC
+            // e.g. 2023-01-01T12:00:00.000Z -> 2023-01-01 12:00:00+00:00
+            // then convert to db timezone
+            val = knex.raw(`? AT TIME ZONE CURRENT_SETTING('timezone')`, [
+              dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ'),
+            ]);
+          } else if (isMssql) {
+            // convert ot UTC
+            // e.g. 2023-05-10T08:49:32.000Z -> 2023-05-10 08:49:32-08:00
+            // then convert to db timezone
+            val = knex.raw(
+              `SWITCHOFFSET(CONVERT(datetimeoffset, ?), DATENAME(TzOffset, SYSDATETIMEOFFSET()))`,
+              [dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ')],
+            );
+          } else {
+            // e.g. 2023-01-01T12:00:00.000Z -> 2023-01-01 12:00:00+00:00
+            val = dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ');
+          }
+        }
         insertObj[sanitize(col.column_name)] = val;
       }
     }

packages/nocodb/src/modules/datas/datas.module.ts

@@ -27,14 +27,18 @@ import { PublicDatasService } from '../../services/public-datas.service';
     }),
   ],
   controllers: [
-    DatasController,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true'
-    BulkDataAliasController,
+      ? [
-    DataAliasController,
+          DatasController,
-    DataAliasNestedController,
+          BulkDataAliasController,
-    DataAliasExportController,
+          DataAliasController,
-    OldDatasController,
+          DataAliasNestedController,
-    PublicDatasController,
+          DataAliasExportController,
-    PublicDatasExportController,
+          OldDatasController,
+          PublicDatasController,
+          PublicDatasExportController,
+        ]
+      : []),
   ],
   providers: [
     DatasService,

packages/nocodb/src/modules/global/global.module.ts

@@ -1,25 +1,18 @@
 import { Global, Module } from '@nestjs/common';
 import { ExtractJwt } from 'passport-jwt';
-import {
-  AppInitService,
-  appInitServiceProvider,
-} from '../../services/app-init.service';
 import { SocketGateway } from '../../gateways/socket.gateway';
-import { Connection } from '../../connection/connection';
 import { GlobalGuard } from '../../guards/global/global.guard';
 import { MetaService } from '../../meta/meta.service';
-import Noco from '../../Noco';
 import { JwtStrategy } from '../../strategies/jwt.strategy';
 import { UsersService } from '../../services/users/users.service';
+import Noco from '../../Noco';
+import { InitMetaServiceProvider } from './init-meta-service.provider';
 import type { Provider } from '@nestjs/common';
 
 export const JwtStrategyProvider: Provider = {
   provide: JwtStrategy,
-  useFactory: async (
+  useFactory: async (usersService: UsersService, metaService: MetaService) => {
-    usersService: UsersService,
+    const config = metaService.config;
-    appInitService: AppInitService,
-  ) => {
-    const config = appInitService.appConfig;
 
     await Noco.initJwt();
 
@@ -34,29 +27,25 @@ export const JwtStrategyProvider: Provider = {
 
     return new JwtStrategy(options, usersService);
   },
-  inject: [UsersService, AppInitService],
+  inject: [UsersService, MetaService],
 };
 
 @Global()
 @Module({
   imports: [],
   providers: [
-    appInitServiceProvider,
+    InitMetaServiceProvider,
-    Connection,
-    MetaService,
     UsersService,
     JwtStrategyProvider,
     GlobalGuard,
-    SocketGateway,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
   ],
   exports: [
-    AppInitService,
-    Connection,
     MetaService,
     JwtStrategyProvider,
     UsersService,
     GlobalGuard,
-    SocketGateway,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
   ],
 })
 export class GlobalModule {}

packages/nocodb/src/modules/global/init-meta-service.provider.ts

@@ -0,0 +1,65 @@
+import { T } from 'nc-help';
+import { MetaService } from '../../meta/meta.service';
+import Noco from '../../Noco';
+import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
+import NcUpgrader from '../../version-upgrader/NcUpgrader';
+import NocoCache from '../../cache/NocoCache';
+import getInstance from '../../utils/getInstance';
+import initAdminFromEnv from '../../helpers/initAdminFromEnv';
+import { User } from '../../models';
+import { NcConfig, prepareEnv } from '../../utils/nc-config';
+import type { Provider } from '@nestjs/common';
+import type { IEventEmitter } from '../event-emitter/event-emitter.interface';
+
+export const InitMetaServiceProvider: Provider = {
+  // initialize app,
+  // 1. init cache
+  // 2. init db connection and create if not exist
+  // 3. init meta and set to Noco
+  // 4. init jwt
+  // 5. init plugin manager
+  // 6. run upgrader
+  useFactory: async (eventEmitter: IEventEmitter) => {
+    // NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB
+    await prepareEnv();
+
+    const config = await NcConfig.createByEnv();
+
+    // set version
+    process.env.NC_VERSION = '0107004';
+
+    // init cache
+    await NocoCache.init();
+
+    // init meta service
+    const metaService = new MetaService(config);
+    await metaService.init();
+
+    // provide meta and config to Noco
+    Noco._ncMeta = metaService;
+    Noco.config = config;
+    Noco.eventEmitter = eventEmitter;
+
+    // init jwt secret
+    await Noco.initJwt();
+
+    // load super admin user from env if env is set
+    await initAdminFromEnv(metaService);
+
+    // init plugin manager
+    await NcPluginMgrv2.init(Noco.ncMeta);
+    await Noco.loadEEState();
+
+    // run upgrader
+    await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
+
+    T.init({
+      instance: getInstance,
+    });
+    T.emit('evt_app_started', await User.count());
+
+    return metaService;
+  },
+  provide: MetaService,
+  inject: ['IEventEmitter'],
+};

packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@@ -1,12 +1,12 @@
 import { Injectable } from '@nestjs/common';
 import PQueue from 'p-queue';
 import Emittery from 'emittery';
-import { JobStatus, JobTypes } from '../../interface/Jobs';
+import { JobStatus, JobTypes } from '../../../interface/Jobs';
-import { DuplicateProcessor } from './export-import/duplicate.processor';
+import { DuplicateProcessor } from '../jobs/export-import/duplicate.processor';
+import { AtImportProcessor } from '../jobs/at-import/at-import.processor';
 import { JobsEventService } from './jobs-event.service';
-import { AtImportProcessor } from './at-import/at-import.processor';
 
-interface Job {
+export interface Job {
   id: string;
   name: string;
   status: string;
@@ -27,16 +27,12 @@ export class QueueService {
     private readonly atImportProcessor: AtImportProcessor,
   ) {
     this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => {
-      const job = this.queueMemory.find(
+      const job = this.queueMemory.find((job) => job.id === data.job.id);
-        (job) => job.id === data.job.id && job.name === data.job.name,
-      );
       job.status = JobStatus.ACTIVE;
       this.jobsEventService.onActive.apply(this.jobsEventService, [job as any]);
     });
     this.emitter.on(JobStatus.COMPLETED, (data: { job: Job; result: any }) => {
-      const job = this.queueMemory.find(
+      const job = this.queueMemory.find((job) => job.id === data.job.id);
-        (job) => job.id === data.job.id && job.name === data.job.name,
-      );
       job.status = JobStatus.COMPLETED;
       this.jobsEventService.onCompleted.apply(this.jobsEventService, [
         job,
@@ -46,9 +42,7 @@ export class QueueService {
       this.removeJob(job);
     });
     this.emitter.on(JobStatus.FAILED, (data: { job: Job; error: Error }) => {
-      const job = this.queueMemory.find(
+      const job = this.queueMemory.find((job) => job.id === data.job.id);
-        (job) => job.id === data.job.id && job.name === data.job.name,
-      );
       job.status = JobStatus.FAILED;
       this.jobsEventService.onFailed.apply(this.jobsEventService, [
         job,
@@ -126,9 +120,7 @@ export class QueueService {
 
   // remove job from memory
   private removeJob(job: Job) {
-    const fIndex = this.queueMemory.findIndex(
+    const fIndex = this.queueMemory.findIndex((q) => q.id === job.id);
-      (q) => q.id === job.id && q.name === job.name,
-    );
     if (fIndex) {
       this.queueMemory.splice(fIndex, 1);
     }

packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts

@@ -7,7 +7,7 @@ import {
 import { Job } from 'bull';
 import boxen from 'boxen';
 import { EventEmitter2 } from '@nestjs/event-emitter';
-import { JobEvents, JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
+import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
 
 @Processor(JOBS_QUEUE)
 export class JobsEventService {
@@ -16,7 +16,6 @@ export class JobsEventService {
   @OnQueueActive()
   onActive(job: Job) {
     this.eventEmitter.emit(JobEvents.STATUS, {
-      name: job.name,
       id: job.id.toString(),
       status: JobStatus.ACTIVE,
     });
@@ -26,7 +25,7 @@ export class JobsEventService {
   onFailed(job: Job, error: Error) {
     console.error(
       boxen(
-        `---- !! JOB FAILED !! ----\nname: ${job.name}\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
+        `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
         {
           padding: 1,
           borderStyle: 'double',
@@ -36,7 +35,6 @@ export class JobsEventService {
     );
 
     this.eventEmitter.emit(JobEvents.STATUS, {
-      name: job.name,
       id: job.id.toString(),
       status: JobStatus.FAILED,
       data: {
@@ -50,7 +48,6 @@ export class JobsEventService {
   @OnQueueCompleted()
   onCompleted(job: Job, data: any) {
     this.eventEmitter.emit(JobEvents.STATUS, {
-      name: job.name,
       id: job.id.toString(),
       status: JobStatus.COMPLETED,
       data: {
@@ -58,12 +55,4 @@ export class JobsEventService {
       },
     });
   }
-
-  sendLog(job: Job, data: { message: string }) {
-    this.eventEmitter.emit(JobEvents.LOG, {
-      name: job.name,
-      id: job.id.toString(),
-      data,
-    });
-  }
 }

packages/nocodb/src/modules/jobs/fallback/jobs.service.ts

@@ -0,0 +1,51 @@
+import { Injectable } from '@nestjs/common';
+import { JobStatus } from '../../../interface/Jobs';
+import { QueueService } from './fallback-queue.service';
+
+@Injectable()
+export class JobsService {
+  constructor(private readonly fallbackQueueService: QueueService) {}
+
+  async add(name: string, data: any) {
+    return this.fallbackQueueService.add(name, data);
+  }
+
+  async jobStatus(jobId: string) {
+    return await (
+      await this.fallbackQueueService.getJob(jobId)
+    ).status;
+  }
+
+  async jobList() {
+    return await this.fallbackQueueService.getJobs([
+      JobStatus.ACTIVE,
+      JobStatus.WAITING,
+      JobStatus.DELAYED,
+      JobStatus.PAUSED,
+    ]);
+  }
+
+  async getJobWithData(data: any) {
+    const jobs = await this.fallbackQueueService.getJobs([
+      // 'completed',
+      JobStatus.WAITING,
+      JobStatus.ACTIVE,
+      JobStatus.DELAYED,
+      // 'failed',
+      JobStatus.PAUSED,
+    ]);
+
+    const job = jobs.find((j) => {
+      for (const key in data) {
+        if (j.data[key]) {
+          if (j.data[key] !== data[key]) return false;
+        } else {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    return job;
+  }
+}

packages/nocodb/src/modules/jobs/jobs.gateway.ts

@@ -9,10 +9,10 @@ import { Server, Socket } from 'socket.io';
 import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
 import { AuthGuard } from '@nestjs/passport';
 import { OnEvent } from '@nestjs/event-emitter';
+import { Inject } from '@nestjs/common';
 import { JobEvents } from '../../interface/Jobs';
-import { JobsService } from './jobs.service';
-import type { JobStatus } from '../../interface/Jobs';
 import type { OnModuleInit } from '@nestjs/common';
+import type { JobStatus } from '../../interface/Jobs';
 
 @WebSocketGateway({
   cors: {
@@ -23,7 +23,7 @@ import type { OnModuleInit } from '@nestjs/common';
   namespace: 'jobs',
 })
 export class JobsGateway implements OnModuleInit {
-  constructor(private readonly jobsService: JobsService) {}
+  constructor(@Inject('JobsService') private readonly jobsService) {}
 
   @WebSocketServer()
   server: Server;
@@ -43,34 +43,28 @@ export class JobsGateway implements OnModuleInit {
   @SubscribeMessage('subscribe')
   async subscribe(
     @MessageBody()
-    body: { _id: number; data: { id: string; name: string } | any },
+    body: { _id: number; data: { id: string } | any },
     @ConnectedSocket() client: Socket,
   ): Promise<void> {
     const { _id, data } = body;
-    if (
+    if (Object.keys(data).every((k) => ['id'].includes(k)) && data?.id) {
-      Object.keys(data).every((k) => ['name', 'id'].includes(k)) &&
+      const rooms = (await this.jobsService.jobList()).map(
-      data?.name &&
+        (j) => `jobs-${j.id}`,
-      data?.id
-    ) {
-      const rooms = (await this.jobsService.jobList(data.name)).map(
-        (j) => `${j.name}-${j.id}`,
       );
-      const room = rooms.find((r) => r === `${data.name}-${data.id}`);
+      const room = rooms.find((r) => r === `jobs-${data.id}`);
       if (room) {
-        client.join(`${data.name}-${data.id}`);
+        client.join(`jobs-${data.id}`);
         client.emit('subscribed', {
           _id,
-          name: data.name,
           id: data.id,
         });
       }
     } else {
       const job = await this.jobsService.getJobWithData(data);
       if (job) {
-        client.join(`${job.name}-${job.id}`);
+        client.join(`jobs-${job.id}`);
         client.emit('subscribed', {
           _id,
-          name: job.name,
           id: job.id,
         });
       }
@@ -79,42 +73,30 @@ export class JobsGateway implements OnModuleInit {
 
   @SubscribeMessage('status')
   async status(
-    @MessageBody() body: { _id: number; data: { id: string; name: string } },
+    @MessageBody() body: { _id: number; data: { id: string } },
     @ConnectedSocket() client: Socket,
   ): Promise<void> {
     const { _id, data } = body;
     client.emit('status', {
       _id,
       id: data.id,
-      name: data.name,
       status: await this.jobsService.jobStatus(data.id),
     });
   }
 
   @OnEvent(JobEvents.STATUS)
-  async sendJobStatus(data: {
+  sendJobStatus(data: { id: string; status: JobStatus; data?: any }): void {
-    name: string;
+    this.server.to(`jobs-${data.id}`).emit('status', {
-    id: string;
-    status: JobStatus;
-    data?: any;
-  }): Promise<void> {
-    this.server.to(`${data.name}-${data.id}`).emit('status', {
       id: data.id,
-      name: data.name,
       status: data.status,
       data: data.data,
     });
   }
 
   @OnEvent(JobEvents.LOG)
-  async sendJobLog(data: {
+  sendJobLog(data: { id: string; data: { message: string } }): void {
-    name: string;
+    this.server.to(`jobs-${data.id}`).emit('log', {
-    id: string;
-    data: { message: string };
-  }): Promise<void> {
-    this.server.to(`${data.name}-${data.id}`).emit('log', {
       id: data.id,
-      name: data.name,
       data: data.data,
     });
   }

packages/nocodb/src/modules/jobs/jobs.module.ts

@@ -4,35 +4,59 @@ import { GlobalModule } from '../global/global.module';
 import { DatasModule } from '../datas/datas.module';
 import { MetasModule } from '../metas/metas.module';
 import { JOBS_QUEUE } from '../../interface/Jobs';
-import { JobsService } from './jobs.service';
+import { ExportService } from './jobs/export-import/export.service';
-import { ExportService } from './export-import/export.service';
+import { ImportService } from './jobs/export-import/import.service';
-import { ImportService } from './export-import/import.service';
+import { AtImportController } from './jobs/at-import/at-import.controller';
-import { DuplicateController } from './export-import/duplicate.controller';
+import { AtImportProcessor } from './jobs/at-import/at-import.processor';
-import { DuplicateProcessor } from './export-import/duplicate.processor';
+import { DuplicateController } from './jobs/export-import/duplicate.controller';
+import { DuplicateProcessor } from './jobs/export-import/duplicate.processor';
+import { JobsLogService } from './jobs/jobs-log.service';
 import { JobsGateway } from './jobs.gateway';
-import { QueueService } from './fallback-queue.service';
+
-import { JobsEventService } from './jobs-event.service';
+// Redis
-import { AtImportController } from './at-import/at-import.controller';
+import { JobsService } from './redis/jobs.service';
-import { AtImportProcessor } from './at-import/at-import.processor';
+import { JobsRedisService } from './redis/jobs-redis.service';
+import { JobsEventService } from './redis/jobs-event.service';
+
+// Fallback
+import { JobsService as FallbackJobsService } from './fallback/jobs.service';
+import { QueueService as FallbackQueueService } from './fallback/fallback-queue.service';
+import { JobsEventService as FallbackJobsEventService } from './fallback/jobs-event.service';
 
 @Module({
   imports: [
     GlobalModule,
     DatasModule,
     MetasModule,
-    BullModule.registerQueue({
+    ...(process.env.NC_REDIS_URL
-      name: JOBS_QUEUE,
+      ? [
-    }),
+          BullModule.forRoot({
+            url: process.env.NC_REDIS_URL,
+          }),
+          BullModule.registerQueue({
+            name: JOBS_QUEUE,
+          }),
+        ]
+      : []),
+  ],
+  controllers: [
+    ...(process.env.NC_WORKER_CONTAINER !== 'true'
+      ? [DuplicateController, AtImportController]
+      : []),
   ],
-  controllers: [DuplicateController, AtImportController],
   providers: [
-    QueueService,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [JobsGateway] : []),
-    JobsGateway,
+    ...(process.env.NC_REDIS_URL
-    JobsService,
+      ? [JobsRedisService, JobsEventService]
-    JobsEventService,
+      : [FallbackQueueService, FallbackJobsEventService]),
-    DuplicateProcessor,
+    {
+      provide: 'JobsService',
+      useClass: process.env.NC_REDIS_URL ? JobsService : FallbackJobsService,
+    },
+    JobsLogService,
     ExportService,
     ImportService,
+    DuplicateProcessor,
     AtImportProcessor,
   ],
 })

packages/nocodb/src/modules/jobs/jobs.service.ts

@@ -1,59 +0,0 @@
-import { InjectQueue } from '@nestjs/bull';
-import { Injectable } from '@nestjs/common';
-import { Queue } from 'bull';
-import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
-import { QueueService } from './fallback-queue.service';
-
-@Injectable()
-export class JobsService {
-  public activeQueue;
-  constructor(
-    @InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
-    private readonly fallbackQueueService: QueueService,
-  ) {
-    this.activeQueue = this.fallbackQueueService;
-    /* process.env.NC_REDIS_URL
-        ? this.jobsQueue
-        : this.fallbackQueueService;
-    */
-  }
-
-  async jobStatus(jobId: string) {
-    return await (await this.activeQueue.getJob(jobId)).getState();
-  }
-
-  async jobList(jobType: string) {
-    return (
-      await this.activeQueue.getJobs([
-        JobStatus.ACTIVE,
-        JobStatus.WAITING,
-        JobStatus.DELAYED,
-        JobStatus.PAUSED,
-      ])
-    ).filter((j) => j.name === jobType);
-  }
-
-  async getJobWithData(data: any) {
-    const jobs = await this.activeQueue.getJobs([
-      // 'completed',
-      JobStatus.WAITING,
-      JobStatus.ACTIVE,
-      JobStatus.DELAYED,
-      // 'failed',
-      JobStatus.PAUSED,
-    ]);
-
-    const job = jobs.find((j) => {
-      for (const key in data) {
-        if (j.data[key]) {
-          if (j.data[key] !== data[key]) return false;
-        } else {
-          return false;
-        }
-      }
-      return true;
-    });
-
-    return job;
-  }
-}

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts

@@ -1,30 +1,36 @@
-import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
+import {
-import { GlobalGuard } from '../../../guards/global/global.guard';
+  Controller,
-import { ExtractProjectIdMiddleware } from '../../../middlewares/extract-project-id/extract-project-id.middleware';
+  HttpCode,
-import { SyncSource } from '../../../models';
+  Inject,
-import { NcError } from '../../../helpers/catchError';
+  Post,
-import { JobsService } from '../jobs.service';
+  Request,
-import { JobTypes } from '../../../interface/Jobs';
+  UseGuards,
+} from '@nestjs/common';
+import { GlobalGuard } from '../../../../guards/global/global.guard';
+import { ExtractProjectIdMiddleware } from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
+import { SyncSource } from '../../../../models';
+import { NcError } from '../../../../helpers/catchError';
+import { JobTypes } from '../../../../interface/Jobs';
 
 @Controller()
 @UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
 export class AtImportController {
-  constructor(private readonly jobsService: JobsService) {}
+  constructor(@Inject('JobsService') private readonly jobsService) {}
 
   @Post('/api/v1/db/meta/import/airtable')
   @HttpCode(200)
   async importAirtable(@Request() req) {
-    const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
+    const job = await this.jobsService.add(JobTypes.AtImport, {
       ...req.body,
     });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 
   @Post('/api/v1/db/meta/syncs/:syncId/trigger')
   @HttpCode(200)
   async triggerSync(@Request() req) {
-    const jobs = await this.jobsService.jobList(JobTypes.AtImport);
+    const jobs = await this.jobsService.jobList();
     const fnd = jobs.find((j) => j.data.syncId === req.params.syncId);
 
     if (fnd) {
@@ -44,7 +50,7 @@ export class AtImportController {
       baseURL = `http://localhost:${process.env.PORT || 8080}`;
     }
 
-    const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
+    const job = await this.jobsService.add(JobTypes.AtImport, {
       syncId: req.params.syncId,
       ...(syncSource?.details || {}),
       projectId: syncSource.project_id,
@@ -54,7 +60,7 @@ export class AtImportController {
       user: user,
     });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 
   @Post('/api/v1/db/meta/syncs/:syncId/abort')

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -9,23 +9,23 @@ import utc from 'dayjs/plugin/utc';
 import tinycolor from 'tinycolor2';
 import { Process, Processor } from '@nestjs/bull';
 import { Job } from 'bull';
-import extractRolesObj from '../../../utils/extractRolesObj';
+import extractRolesObj from '../../../../utils/extractRolesObj';
-import { AttachmentsService } from '../../../services/attachments.service';
+import { AttachmentsService } from '../../../../services/attachments.service';
-import { ColumnsService } from '../../../services/columns.service';
+import { ColumnsService } from '../../../../services/columns.service';
-import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
-import { FiltersService } from '../../../services/filters.service';
+import { FiltersService } from '../../../../services/filters.service';
-import { FormColumnsService } from '../../../services/form-columns.service';
+import { FormColumnsService } from '../../../../services/form-columns.service';
-import { GalleriesService } from '../../../services/galleries.service';
+import { GalleriesService } from '../../../../services/galleries.service';
-import { GridsService } from '../../../services/grids.service';
+import { GridsService } from '../../../../services/grids.service';
-import { ProjectUsersService } from '../../../services/project-users/project-users.service';
+import { ProjectUsersService } from '../../../../services/project-users/project-users.service';
-import { ProjectsService } from '../../../services/projects.service';
+import { ProjectsService } from '../../../../services/projects.service';
-import { SortsService } from '../../../services/sorts.service';
+import { SortsService } from '../../../../services/sorts.service';
-import { TablesService } from '../../../services/tables.service';
+import { TablesService } from '../../../../services/tables.service';
-import { ViewColumnsService } from '../../../services/view-columns.service';
+import { ViewColumnsService } from '../../../../services/view-columns.service';
-import { ViewsService } from '../../../services/views.service';
+import { ViewsService } from '../../../../services/views.service';
-import { FormsService } from '../../../services/forms.service';
+import { FormsService } from '../../../../services/forms.service';
-import { JobsEventService } from '../jobs-event.service';
+import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
-import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
+import { JobsLogService } from '../jobs-log.service';
 import FetchAT from './helpers/fetchAT';
 import { importData, importLTARData } from './helpers/readAndProcessData';
 import EntityMap from './helpers/EntityMap';
@@ -99,7 +99,7 @@ export class AtImportProcessor {
     private readonly viewColumnsService: ViewColumnsService,
     private readonly sortsService: SortsService,
     private readonly bulkDataAliasService: BulkDataAliasService,
-    private readonly jobsEventService: JobsEventService,
+    private readonly jobsLogService: JobsLogService,
   ) {}
 
   @Process(JobTypes.AtImport)
@@ -135,11 +135,11 @@ export class AtImportProcessor {
     };
 
     const logBasic = (log) => {
-      this.jobsEventService.sendLog(job, { message: log });
+      this.jobsLogService.sendLog(job, { message: log });
     };
 
     const logDetailed = (log) => {
-      if (debugMode) this.jobsEventService.sendLog(job, { message: log });
+      if (debugMode) this.jobsLogService.sendLog(job, { message: log });
     };
 
     const perfStats = [];

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@@ -1,8 +1,8 @@
 /* eslint-disable no-async-promise-executor */
 import { RelationTypes, UITypes } from 'nocodb-sdk';
 import EntityMap from './EntityMap';
-import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
+import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service';
-import type { TablesService } from '../../../../services/tables.service';
+import type { TablesService } from '../../../../../services/tables.service';
 // @ts-ignore
 import type { AirtableBase } from 'airtable/lib/airtable_base';
 import type { TableType } from 'nocodb-sdk';

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@@ -2,28 +2,28 @@ import {
   Body,
   Controller,
   HttpCode,
+  Inject,
   Param,
   Post,
   Request,
   UseGuards,
 } from '@nestjs/common';
 import { ProjectStatus } from 'nocodb-sdk';
-import { GlobalGuard } from '../../../guards/global/global.guard';
+import { GlobalGuard } from '../../../../guards/global/global.guard';
 import {
   Acl,
   ExtractProjectIdMiddleware,
-} from '../../../middlewares/extract-project-id/extract-project-id.middleware';
+} from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
-import { ProjectsService } from '../../../services/projects.service';
+import { ProjectsService } from '../../../../services/projects.service';
-import { Base, Model, Project } from '../../../models';
+import { Base, Model, Project } from '../../../../models';
-import { generateUniqueName } from '../../../helpers/exportImportHelpers';
+import { generateUniqueName } from '../../../../helpers/exportImportHelpers';
-import { JobsService } from '../jobs.service';
+import { JobTypes } from '../../../../interface/Jobs';
-import { JobTypes } from '../../../interface/Jobs';
 
 @Controller()
 @UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
 export class DuplicateController {
   constructor(
-    private readonly jobsService: JobsService,
+    @Inject('JobsService') private readonly jobsService,
     private readonly projectsService: ProjectsService,
   ) {}
 
@@ -35,10 +35,14 @@ export class DuplicateController {
     @Param('projectId') projectId: string,
     @Param('baseId') baseId?: string,
     @Body()
-    options?: {
+    body?: {
-      excludeData?: boolean;
+      options?: {
-      excludeViews?: boolean;
+        excludeData?: boolean;
-      excludeHooks?: boolean;
+        excludeViews?: boolean;
+        excludeHooks?: boolean;
+      };
+      // override duplicated project
+      project?: any;
     },
   ) {
     const project = await Project.get(projectId);
@@ -63,22 +67,26 @@ export class DuplicateController {
     );
 
     const dupProject = await this.projectsService.projectCreate({
-      project: { title: uniqueTitle, status: ProjectStatus.JOB },
+      project: {
+        title: uniqueTitle,
+        status: ProjectStatus.JOB,
+        ...(body.project || {}),
+      },
       user: { id: req.user.id },
     });
 
-    const job = await this.jobsService.activeQueue.add(JobTypes.DuplicateBase, {
+    const job = await this.jobsService.add(JobTypes.DuplicateBase, {
       projectId: project.id,
       baseId: base.id,
       dupProjectId: dupProject.id,
-      options,
+      options: body.options || {},
       req: {
         user: req.user,
         clientIp: req.clientIp,
       },
     });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 
   @Post('/api/v1/db/meta/duplicate/:projectId/table/:modelId')
@@ -89,10 +97,12 @@ export class DuplicateController {
     @Param('projectId') projectId: string,
     @Param('modelId') modelId?: string,
     @Body()
-    options?: {
+    body?: {
-      excludeData?: boolean;
+      options?: {
-      excludeViews?: boolean;
+        excludeData?: boolean;
-      excludeHooks?: boolean;
+        excludeViews?: boolean;
+        excludeHooks?: boolean;
+      };
     },
   ) {
     const project = await Project.get(projectId);
@@ -116,21 +126,18 @@ export class DuplicateController {
       models.map((p) => p.title),
     );
 
-    const job = await this.jobsService.activeQueue.add(
+    const job = await this.jobsService.add(JobTypes.DuplicateModel, {
-      JobTypes.DuplicateModel,
+      projectId: project.id,
-      {
+      baseId: base.id,
-        projectId: project.id,
+      modelId: model.id,
-        baseId: base.id,
+      title: uniqueTitle,
-        modelId: model.id,
+      options: body.options || {},
-        title: uniqueTitle,
+      req: {
-        options,
+        user: req.user,
-        req: {
+        clientIp: req.clientIp,
-          user: req.user,
-          clientIp: req.clientIp,
-        },
       },
-    );
+    });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 }

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@@ -4,12 +4,12 @@ import { Job } from 'bull';
 import papaparse from 'papaparse';
 import { UITypes } from 'nocodb-sdk';
 import { Logger } from '@nestjs/common';
-import { Base, Column, Model, Project } from '../../../models';
+import { Base, Column, Model, Project } from '../../../../models';
-import { ProjectsService } from '../../../services/projects.service';
+import { ProjectsService } from '../../../../services/projects.service';
-import { findWithIdentifier } from '../../../helpers/exportImportHelpers';
+import { findWithIdentifier } from '../../../../helpers/exportImportHelpers';
-import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
-import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
+import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
-import { elapsedTime, initTime } from '../helpers';
+import { elapsedTime, initTime } from '../../helpers';
 import { ExportService } from './export.service';
 import { ImportService } from './import.service';
 

packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@@ -2,19 +2,19 @@ import { Readable } from 'stream';
 import { UITypes, ViewTypes } from 'nocodb-sdk';
 import { unparse } from 'papaparse';
 import { Injectable, Logger } from '@nestjs/common';
-import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
+import NcConnectionMgrv2 from '../../../../utils/common/NcConnectionMgrv2';
-import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers';
+import { getViewAndModelByAliasOrId } from '../../../datas/helpers';
 import {
   clearPrefix,
   generateBaseIdMap,
-} from '../../../helpers/exportImportHelpers';
+} from '../../../../helpers/exportImportHelpers';
-import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
+import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
-import { NcError } from '../../../helpers/catchError';
+import { NcError } from '../../../../helpers/catchError';
-import { Base, Hook, Model, Project } from '../../../models';
+import { Base, Hook, Model, Project } from '../../../../models';
-import { DatasService } from '../../../services/datas.service';
+import { DatasService } from '../../../../services/datas.service';
-import { elapsedTime, initTime } from '../helpers';
+import { elapsedTime, initTime } from '../../helpers';
-import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2';
+import type { BaseModelSqlv2 } from '../../../../db/BaseModelSqlv2';
-import type { View } from '../../../models';
+import type { View } from '../../../../models';
 
 @Injectable()
 export class ExportService {

packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@@ -9,28 +9,28 @@ import {
   reverseGet,
   withoutId,
   withoutNull,
-} from '../../../helpers/exportImportHelpers';
+} from '../../../../helpers/exportImportHelpers';
-import { NcError } from '../../../helpers/catchError';
+import { NcError } from '../../../../helpers/catchError';
-import { Base, Column, Model, Project } from '../../../models';
+import { Base, Column, Model, Project } from '../../../../models';
-import { TablesService } from '../../../services/tables.service';
+import { TablesService } from '../../../../services/tables.service';
-import { ColumnsService } from '../../../services/columns.service';
+import { ColumnsService } from '../../../../services/columns.service';
-import { FiltersService } from '../../../services/filters.service';
+import { FiltersService } from '../../../../services/filters.service';
-import { SortsService } from '../../../services/sorts.service';
+import { SortsService } from '../../../../services/sorts.service';
-import { ViewColumnsService } from '../../../services/view-columns.service';
+import { ViewColumnsService } from '../../../../services/view-columns.service';
-import { GridColumnsService } from '../../../services/grid-columns.service';
+import { GridColumnsService } from '../../../../services/grid-columns.service';
-import { FormColumnsService } from '../../../services/form-columns.service';
+import { FormColumnsService } from '../../../../services/form-columns.service';
-import { GridsService } from '../../../services/grids.service';
+import { GridsService } from '../../../../services/grids.service';
-import { FormsService } from '../../../services/forms.service';
+import { FormsService } from '../../../../services/forms.service';
-import { GalleriesService } from '../../../services/galleries.service';
+import { GalleriesService } from '../../../../services/galleries.service';
-import { KanbansService } from '../../../services/kanbans.service';
+import { KanbansService } from '../../../../services/kanbans.service';
-import { HooksService } from '../../../services/hooks.service';
+import { HooksService } from '../../../../services/hooks.service';
-import { ViewsService } from '../../../services/views.service';
+import { ViewsService } from '../../../../services/views.service';
-import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
+import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
-import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
-import { elapsedTime, initTime } from '../helpers';
+import { elapsedTime, initTime } from '../../helpers';
 import type { Readable } from 'stream';
 import type { ViewCreateReqType } from 'nocodb-sdk';
-import type { LinkToAnotherRecordColumn, User, View } from '../../../models';
+import type { LinkToAnotherRecordColumn, User, View } from '../../../../models';
 
 @Injectable()
 export class ImportService {

packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { JobEvents } from '../../../interface/Jobs';
+import type { Job } from 'bull';
+
+@Injectable()
+export class JobsLogService {
+  constructor(private eventEmitter: EventEmitter2) {}
+
+  sendLog(job: Job, data: { message: string }) {
+    this.eventEmitter.emit(JobEvents.LOG, {
+      id: job.id.toString(),
+      data,
+    });
+  }
+}

packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts

@@ -0,0 +1,107 @@
+import {
+  OnQueueActive,
+  OnQueueCompleted,
+  OnQueueFailed,
+  Processor,
+} from '@nestjs/bull';
+import { Job } from 'bull';
+import boxen from 'boxen';
+import { EventEmitter2, OnEvent } from '@nestjs/event-emitter';
+import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
+import { JobsRedisService } from './jobs-redis.service';
+
+@Processor(JOBS_QUEUE)
+export class JobsEventService {
+  constructor(
+    private jobsRedisService: JobsRedisService,
+    private eventEmitter: EventEmitter2,
+  ) {}
+
+  @OnQueueActive()
+  onActive(job: Job) {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
+        cmd: JobEvents.STATUS,
+        id: job.id.toString(),
+        status: JobStatus.ACTIVE,
+      });
+    } else {
+      this.eventEmitter.emit(JobEvents.STATUS, {
+        id: job.id.toString(),
+        status: JobStatus.ACTIVE,
+      });
+    }
+  }
+
+  @OnQueueFailed()
+  onFailed(job: Job, error: Error) {
+    console.error(
+      boxen(
+        `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
+        {
+          padding: 1,
+          borderStyle: 'double',
+          borderColor: 'yellow',
+        },
+      ),
+    );
+
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
+        cmd: JobEvents.STATUS,
+        id: job.id.toString(),
+        status: JobStatus.FAILED,
+        data: {
+          error: {
+            message: error?.message,
+          },
+        },
+      });
+    } else {
+      this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
+      this.eventEmitter.emit(JobEvents.STATUS, {
+        id: job.id.toString(),
+        status: JobStatus.FAILED,
+        data: {
+          error: {
+            message: error?.message,
+          },
+        },
+      });
+    }
+  }
+
+  @OnQueueCompleted()
+  onCompleted(job: Job, data: any) {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
+        cmd: JobEvents.STATUS,
+        id: job.id.toString(),
+        status: JobStatus.COMPLETED,
+        data: {
+          result: data,
+        },
+      });
+    } else {
+      this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
+      this.eventEmitter.emit(JobEvents.STATUS, {
+        id: job.id.toString(),
+        status: JobStatus.COMPLETED,
+        data: {
+          result: data,
+        },
+      });
+    }
+  }
+
+  @OnEvent(JobEvents.LOG)
+  onLog(data: { id: string; data: { message: string } }) {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${data.id}`, {
+        cmd: JobEvents.LOG,
+        id: data.id,
+        data: data.data,
+      });
+    }
+  }
+}

packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts

@@ -0,0 +1,53 @@
+import { Injectable } from '@nestjs/common';
+import Redis from 'ioredis';
+
+@Injectable()
+export class JobsRedisService {
+  private redisClient: Redis;
+  private redisSubscriber: Redis;
+  private unsubscribeCallbacks: { [key: string]: () => void } = {};
+
+  constructor() {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.redisClient = new Redis(process.env.NC_REDIS_URL);
+      return;
+    }
+    this.redisSubscriber = new Redis(process.env.NC_REDIS_URL);
+  }
+
+  publish(channel: string, message: string | any) {
+    if (typeof message === 'string') {
+      this.redisClient.publish(channel, message);
+    } else {
+      try {
+        this.redisClient.publish(channel, JSON.stringify(message));
+      } catch (e) {
+        console.error(e);
+      }
+    }
+  }
+
+  subscribe(channel: string, callback: (message: any) => void) {
+    this.redisSubscriber.subscribe(channel);
+
+    const onMessage = (_channel, message) => {
+      try {
+        message = JSON.parse(message);
+      } catch (e) {}
+      callback(message);
+    };
+
+    this.redisSubscriber.on('message', onMessage);
+    this.unsubscribeCallbacks[channel] = () => {
+      this.redisSubscriber.unsubscribe(channel);
+      this.redisSubscriber.off('message', onMessage);
+    };
+  }
+
+  unsubscribe(channel: string) {
+    if (this.unsubscribeCallbacks[channel]) {
+      this.unsubscribeCallbacks[channel]();
+      delete this.unsubscribeCallbacks[channel];
+    }
+  }
+}

packages/nocodb/src/modules/jobs/redis/jobs.service.ts

@@ -0,0 +1,98 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable } from '@nestjs/common';
+import { Queue } from 'bull';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
+import { JobsRedisService } from './jobs-redis.service';
+import type { OnModuleInit } from '@nestjs/common';
+
+@Injectable()
+export class JobsService implements OnModuleInit {
+  constructor(
+    @InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
+    private jobsRedisService: JobsRedisService,
+    private eventEmitter: EventEmitter2,
+  ) {}
+
+  // pause primary instance queue
+  async onModuleInit() {
+    if (process.env.NC_WORKER_CONTAINER !== 'true') {
+      await this.jobsQueue.pause(true);
+    }
+  }
+
+  async add(name: string, data: any) {
+    // resume primary instance queue if there is no worker
+    const workerCount = (await this.jobsQueue.getWorkers()).length;
+    const localWorkerPaused = await this.jobsQueue.isPaused(true);
+
+    // if there is no worker and primary instance queue is paused, resume it
+    // if there is any worker and primary instance queue is not paused, pause it
+    if (workerCount < 1 && localWorkerPaused) {
+      await this.jobsQueue.resume(true);
+    } else if (workerCount > 0 && !localWorkerPaused) {
+      await this.jobsQueue.pause(true);
+    }
+
+    const job = await this.jobsQueue.add(name, data);
+
+    // subscribe to job events
+    this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => {
+      const cmd = data.cmd;
+      delete data.cmd;
+      switch (cmd) {
+        case JobEvents.STATUS:
+          this.eventEmitter.emit(JobEvents.STATUS, data);
+          if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) {
+            this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`);
+          }
+          break;
+        case JobEvents.LOG:
+          this.eventEmitter.emit(JobEvents.LOG, data);
+          break;
+      }
+    });
+
+    return job;
+  }
+
+  async jobStatus(jobId: string) {
+    const job = await this.jobsQueue.getJob(jobId);
+    if (job) {
+      return await job.getState();
+    }
+  }
+
+  async jobList() {
+    return await this.jobsQueue.getJobs([
+      JobStatus.ACTIVE,
+      JobStatus.WAITING,
+      JobStatus.DELAYED,
+      JobStatus.PAUSED,
+    ]);
+  }
+
+  async getJobWithData(data: any) {
+    const jobs = await this.jobsQueue.getJobs([
+      // 'completed',
+      JobStatus.WAITING,
+      JobStatus.ACTIVE,
+      JobStatus.DELAYED,
+      // 'failed',
+      JobStatus.PAUSED,
+    ]);
+
+    const job = jobs.find((j) => {
+      for (const key in data) {
+        if (j.data[key]) {
+          if (j.data[key] !== data[key]) return false;
+        } else {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    return job;
+  }
+}

packages/nocodb/src/modules/metas/metas.module.ts

@@ -67,7 +67,6 @@ import { UtilsService } from '../../services/utils.service';
 import { ViewColumnsService } from '../../services/view-columns.service';
 import { ViewsService } from '../../services/views.service';
 import { ApiDocsService } from '../../services/api-docs/api-docs.service';
-import { EventEmitterModule } from '../event-emitter/event-emitter.module';
 import { GlobalModule } from '../global/global.module';
 import { ProjectUsersController } from '../../controllers/project-users.controller';
 import { ProjectUsersService } from '../../services/project-users/project-users.service';
@@ -83,38 +82,42 @@ import { ProjectUsersService } from '../../services/project-users/project-users.
     GlobalModule,
   ],
   controllers: [
-    ApiDocsController,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true'
-    ApiTokensController,
+      ? [
-    AttachmentsController,
+          ApiDocsController,
-    AuditsController,
+          ApiTokensController,
-    BasesController,
+          AttachmentsController,
-    CachesController,
+          AuditsController,
-    ColumnsController,
+          BasesController,
-    FiltersController,
+          CachesController,
-    FormColumnsController,
+          ColumnsController,
-    FormsController,
+          FiltersController,
-    GalleriesController,
+          FormColumnsController,
-    GridColumnsController,
+          FormsController,
-    GridsController,
+          GalleriesController,
-    HooksController,
+          GridColumnsController,
-    KanbansController,
+          GridsController,
-    MapsController,
+          HooksController,
-    MetaDiffsController,
+          KanbansController,
-    ModelVisibilitiesController,
+          MapsController,
-    OrgLcenseController,
+          MetaDiffsController,
-    OrgTokensController,
+          ModelVisibilitiesController,
-    OrgUsersController,
+          OrgLcenseController,
-    PluginsController,
+          OrgTokensController,
-    ProjectUsersController,
+          OrgUsersController,
-    ProjectsController,
+          PluginsController,
-    PublicMetasController,
+          ProjectUsersController,
-    ViewsController,
+          ProjectsController,
-    ViewColumnsController,
+          PublicMetasController,
-    UtilsController,
+          ViewsController,
-    TablesController,
+          ViewColumnsController,
-    SyncController,
+          UtilsController,
-    SortsController,
+          TablesController,
-    SharedBasesController,
+          SyncController,
+          SortsController,
+          SharedBasesController,
+        ]
+      : []),
   ],
   providers: [
     ApiDocsService,

packages/nocodb/src/modules/test/test.module.ts

@@ -2,6 +2,8 @@ import { Module } from '@nestjs/common';
 import { TestController } from '../../controllers/test/test.controller';
 
 @Module({
-  controllers: [TestController],
+  controllers: [
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [TestController] : []),
+  ],
 })
 export class TestModule {}

packages/nocodb/src/modules/users/users.module.ts

@@ -10,7 +10,9 @@ import { UsersController } from '../../controllers/users/users.controller';
 
 @Module({
   imports: [GlobalModule, PassportModule],
-  controllers: [UsersController],
+  controllers: [
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [UsersController] : []),
+  ],
   providers: [UsersService, GoogleStrategyProvider],
   exports: [UsersService],
 })

packages/nocodb/src/plugins/storage/Local.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import { promisify } from 'util';
 import mkdirp from 'mkdirp';
 import axios from 'axios';
-import NcConfigFactory from '../../utils/NcConfigFactory';
+import { getToolDir } from '../../utils/nc-config';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 
@@ -11,7 +11,7 @@ export default class Local implements IStorageAdapterV2 {
   constructor() {}
 
   public async fileCreate(key: string, file: XcFile): Promise<any> {
-    const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const destPath = path.join(getToolDir(), ...key.split('/'));
     try {
       await mkdirp(path.dirname(destPath));
       const data = await promisify(fs.readFile)(file.path);
@@ -24,7 +24,7 @@ export default class Local implements IStorageAdapterV2 {
   }
 
   async fileCreateByUrl(key: string, url: string): Promise<any> {
-    const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const destPath = path.join(getToolDir(), ...key.split('/'));
     return new Promise((resolve, reject) => {
       axios
         .get(url, {
@@ -71,10 +71,7 @@ export default class Local implements IStorageAdapterV2 {
     stream: Readable,
   ): Promise<void> {
     return new Promise((resolve, reject) => {
-      const destPath = path.join(
+      const destPath = path.join(getToolDir(), ...key.split('/'));
-        NcConfigFactory.getToolDir(),
-        ...key.split('/'),
-      );
       try {
         mkdirp(path.dirname(destPath)).then(() => {
           const writableStream = fs.createWriteStream(destPath);
@@ -89,12 +86,12 @@ export default class Local implements IStorageAdapterV2 {
   }
 
   public async fileReadByStream(key: string): Promise<Readable> {
-    const srcPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const srcPath = path.join(getToolDir(), ...key.split('/'));
     return fs.createReadStream(srcPath, { encoding: 'utf8' });
   }
 
   public async getDirectoryList(key: string): Promise<string[]> {
-    const destDir = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const destDir = path.join(getToolDir(), ...key.split('/'));
     return fs.promises.readdir(destDir);
   }
 
@@ -106,7 +103,7 @@ export default class Local implements IStorageAdapterV2 {
   public async fileRead(filePath: string): Promise<any> {
     try {
       const fileData = await fs.promises.readFile(
-        path.join(NcConfigFactory.getToolDir(), ...filePath.split('/')),
+        path.join(getToolDir(), ...filePath.split('/')),
       );
       return fileData;
     } catch (e) {

packages/nocodb/src/schema/swagger.json

@@ -2133,16 +2133,25 @@
               "schema": {
                 "type": "object",
                 "properties": {
-                  "excludeData": {
+                  "options": {
-                    "type": "boolean",
+                    "type": "object",
-                    "required": false
+                    "properties": {
-                  },
+                      "excludeData": {
-                  "excludeViews": {
+                        "type": "boolean",
-                    "type": "boolean",
+                        "required": false
-                    "required": false
+                      },
+                      "excludeViews": {
+                        "type": "boolean",
+                        "required": false
+                      },
+                      "excludeHooks": {
+                        "type": "boolean",
+                        "required": false
+                      }
+                    }
                   },
-                  "excludeHooks": {
+                  "project": {
-                    "type": "boolean",
+                    "type": "object",
                     "required": false
                   }
                 }
@@ -2223,16 +2232,25 @@
               "schema": {
                 "type": "object",
                 "properties": {
-                  "excludeData": {
+                  "options": {
-                    "type": "boolean",
+                    "type": "object",
-                    "required": false
+                    "properties": {
-                  },
+                      "excludeData": {
-                  "excludeViews": {
+                        "type": "boolean",
-                    "type": "boolean",
+                        "required": false
-                    "required": false
+                      },
+                      "excludeViews": {
+                        "type": "boolean",
+                        "required": false
+                      },
+                      "excludeHooks": {
+                        "type": "boolean",
+                        "required": false
+                      }
+                    }
                   },
-                  "excludeHooks": {
+                  "project": {
-                    "type": "boolean",
+                    "type": "object",
                     "required": false
                   }
                 }
@@ -3978,13 +3996,22 @@
               "schema": {
                 "type": "object",
                 "properties": {
-                  "excludeData": {
+                  "options": {
-                    "type": "boolean",
+                    "type": "object",
-                    "required": false
+                    "properties": {
-                  },
+                      "excludeData": {
-                  "excludeViews": {
+                        "type": "boolean",
-                    "type": "boolean",
+                        "required": false
-                    "required": false
+                      },
+                      "excludeViews": {
+                        "type": "boolean",
+                        "required": false
+                      },
+                      "excludeHooks": {
+                        "type": "boolean",
+                        "required": false
+                      }
+                    }
                   }
                 }
               },

packages/nocodb/src/services/app-init.service.spec.ts

@@ -1,19 +0,0 @@
-import { Test } from '@nestjs/testing';
-import { AppInitService } from './app-init.service';
-import type { TestingModule } from '@nestjs/testing';
-
-describe('AppInitService', () => {
-  let service: AppInitService;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [AppInitService],
-    }).compile();
-
-    service = module.get<AppInitService>(AppInitService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

packages/nocodb/src/services/app-init.service.ts

@@ -1,79 +0,0 @@
-import { T } from 'nc-help';
-import NocoCache from '../cache/NocoCache';
-import { Connection } from '../connection/connection';
-import initAdminFromEnv from '../helpers/initAdminFromEnv';
-import NcPluginMgrv2 from '../helpers/NcPluginMgrv2';
-import { MetaService } from '../meta/meta.service';
-import { User } from '../models';
-import Noco from '../Noco';
-import getInstance from '../utils/getInstance';
-import NcConfigFactory from '../utils/NcConfigFactory';
-import NcUpgrader from '../version-upgrader/NcUpgrader';
-import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface';
-import type { Provider } from '@nestjs/common';
-
-export class AppInitService {
-  private readonly config: any;
-
-  constructor(config) {
-    this.config = config;
-  }
-
-  get appConfig(): any {
-    return this.config;
-  }
-}
-
-export const appInitServiceProvider: Provider = {
-  provide: AppInitService,
-  // initialize app,
-  // 1. init cache
-  // 2. init db connection and create if not exist
-  // 3. init meta and set to Noco
-  // 4. init jwt
-  // 5. init plugin manager
-  // 6. run upgrader
-  useFactory: async (
-    connection: Connection,
-    metaService: MetaService,
-    eventEmitter: IEventEmitter,
-  ) => {
-    process.env.NC_VERSION = '0107004';
-
-    await NocoCache.init();
-
-    await connection.init();
-
-    await NcConfigFactory.metaDbCreateIfNotExist(connection.config);
-
-    await metaService.init();
-
-    // todo: remove
-    // temporary hack
-    Noco._ncMeta = metaService;
-    Noco.config = connection.config;
-    Noco.eventEmitter = eventEmitter;
-
-    // init jwt secret
-    await Noco.initJwt();
-
-    // load super admin user from env if env is set
-    await initAdminFromEnv(metaService);
-
-    // init plugin manager
-    await NcPluginMgrv2.init(Noco.ncMeta);
-    await Noco.loadEEState();
-
-    // run upgrader
-    await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
-
-    T.init({
-      instance: getInstance,
-    });
-    T.emit('evt_app_started', await User.count());
-
-    // todo: move app config to app-init service
-    return new AppInitService(connection.config);
-  },
-  inject: [Connection, MetaService, 'IEventEmitter'],
-};

packages/nocodb/src/services/auth.service.ts

@@ -5,18 +5,13 @@ import * as bcrypt from 'bcryptjs';
 
 import { v4 as uuidv4 } from 'uuid';
 import Noco from '../Noco';
-import { Connection } from '../connection/connection';
 import { genJwt } from './users/helpers';
 import { UsersService } from './users/users.service';
 import type { CreateUserDto } from '../controllers/auth.controller';
 
 @Injectable()
 export class AuthService {
-  constructor(
+  constructor(private usersService: UsersService) {}
-    private usersService: UsersService,
-    // private jwtService: JwtService,
-    private connection: Connection,
-  ) {}
 
   async validateUser(email: string, pass: string): Promise<any> {
     const user = await this.usersService.findOne(email);

packages/nocodb/src/services/projects.service.ts

@@ -11,7 +11,7 @@ import syncMigration from '../helpers/syncMigration';
 import { Project, ProjectUser } from '../models';
 import Noco from '../Noco';
 import extractRolesObj from '../utils/extractRolesObj';
-import NcConfigFactory from '../utils/NcConfigFactory';
+import { getToolDir } from '../utils/nc-config';
 import type { ProjectUpdateReqType } from 'nocodb-sdk';
 import type { ProjectReqType } from 'nocodb-sdk';
 
@@ -96,7 +96,7 @@ export class ProjectsService {
         // if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project
         // each file will be named as nc_<random_id>.db
         const fs = require('fs');
-        const toolDir = NcConfigFactory.getToolDir();
+        const toolDir = getToolDir();
         const nanoidv2 = customAlphabet(
           '1234567890abcdefghijklmnopqrstuvwxyz',
           14,

packages/nocodb/src/services/utils.service.ts

@@ -9,7 +9,7 @@ import { Project, User } from '../models';
 import Noco from '../Noco';
 import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
 import { MetaTable } from '../utils/globals';
-import NcConfigFactory from '../utils/NcConfigFactory';
+import { jdbcToXcConfig } from '../utils/nc-config/helpers';
 import { packageVersion } from '../utils/packageVersion';
 
 const versionCache = {
@@ -186,7 +186,7 @@ export class UtilsService {
   }) {
     const { url } = param.body;
     try {
-      const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true);
+      const connectionConfig = jdbcToXcConfig(url);
       return connectionConfig;
     } catch (error) {
       return NcError.internalServerError(

packages/nocodb/src/utils/NcConfigFactory.ts

@@ -1,755 +0,0 @@
-import fs from 'fs';
-import { URL } from 'url';
-import { promisify } from 'util';
-import * as path from 'path';
-import parseDbUrl from 'parse-database-url';
-import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory';
-// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory';
-// import type {
-//   AuthConfig,
-//   DbConfig,
-//   MailerConfig,
-//   NcConfig,
-// } from '../../interface/config';
-
-// const {
-//   uniqueNamesGenerator,
-//   starWars,
-//   adjectives,
-//   animals,
-// } = require('unique-names-generator');
-
-type NcConfig = any;
-type DbConfig = any;
-
-const driverClientMapping = {
-  mysql: 'mysql2',
-  mariadb: 'mysql2',
-  postgres: 'pg',
-  postgresql: 'pg',
-  sqlite: 'sqlite3',
-  mssql: 'mssql',
-};
-
-const defaultClientPortMapping = {
-  mysql: 3306,
-  mysql2: 3306,
-  postgres: 5432,
-  pg: 5432,
-  mssql: 1433,
-};
-
-const defaultConnectionConfig: any = {
-  // https://github.com/knex/knex/issues/97
-  // timezone: process.env.NC_TIMEZONE || 'UTC',
-  dateStrings: true,
-};
-
-// default knex options
-const defaultConnectionOptions = {
-  pool: {
-    min: 0,
-    max: 10,
-  },
-};
-
-const knownQueryParams = [
-  {
-    parameter: 'database',
-    aliases: ['d', 'db'],
-  },
-  {
-    parameter: 'password',
-    aliases: ['p'],
-  },
-  {
-    parameter: 'user',
-    aliases: ['u'],
-  },
-  {
-    parameter: 'title',
-    aliases: ['t'],
-  },
-  {
-    parameter: 'keyFilePath',
-    aliases: [],
-  },
-  {
-    parameter: 'certFilePath',
-    aliases: [],
-  },
-  {
-    parameter: 'caFilePath',
-    aliases: [],
-  },
-  {
-    parameter: 'ssl',
-    aliases: [],
-  },
-  {
-    parameter: 'options',
-    aliases: ['opt', 'opts'],
-  },
-];
-
-export default class NcConfigFactory {
-  public static async make(): Promise<any> {
-    await this.jdbcToXcUrl();
-
-    const ncConfig = new NcConfigFactory();
-
-    ncConfig.auth = {
-      jwt: {
-        secret: process.env.NC_AUTH_JWT_SECRET,
-      },
-    };
-
-    ncConfig.port = +(process?.env?.PORT ?? 8080);
-    ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev';
-    ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev';
-    // ncConfig.toolDir = this.getToolDir();
-    ncConfig.projectType =
-      ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
-
-    if (ncConfig.meta?.db?.connection?.filename) {
-      ncConfig.meta.db.connection.filename = path.join(
-        this.getToolDir(),
-        ncConfig.meta.db.connection.filename,
-      );
-    }
-
-    if (process.env.NC_DB) {
-      ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
-    } else if (process.env.NC_DB_JSON) {
-      ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON);
-    } else if (process.env.NC_DB_JSON_FILE) {
-      const filePath = process.env.NC_DB_JSON_FILE;
-
-      if (!(await promisify(fs.exists)(filePath))) {
-        throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`);
-      }
-
-      const fileContent = await promisify(fs.readFile)(filePath, {
-        encoding: 'utf8',
-      });
-      ncConfig.meta.db = JSON.parse(fileContent);
-    }
-
-    if (process.env.NC_TRY) {
-      ncConfig.try = true;
-      ncConfig.meta.db = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        pool: {
-          min: 1,
-          max: 1,
-          // disposeTimeout: 360000*1000,
-          idleTimeoutMillis: 360000 * 1000,
-        },
-      } as any;
-    }
-
-    if (process.env.NC_PUBLIC_URL) {
-      ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
-      // ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
-      ncConfig.publicUrl = process.env.NC_PUBLIC_URL;
-    }
-
-    if (process.env.NC_DASHBOARD_URL) {
-      ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL;
-    }
-
-    return ncConfig;
-  }
-
-  public static getToolDir() {
-    return process.env.NC_TOOL_DIR || process.cwd();
-  }
-
-  public static hasDbUrl(): boolean {
-    return Object.keys(process.env).some((envKey) =>
-      envKey.startsWith('NC_DB_URL'),
-    );
-  }
-
-  public static makeFromUrls(urls: string[]): NcConfig {
-    const config = new NcConfigFactory();
-
-    // config.envs[process.env.NODE_ENV || 'dev'].db = [];
-    config.envs['_noco'].db = [];
-    for (const [i, url] of Object.entries(urls)) {
-      // config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i));
-      config.envs['_noco'].db.push(this.urlToDbConfig(url, i));
-    }
-
-    return config;
-  }
-
-  public static urlToDbConfig(
-    urlString: string,
-    key = '',
-    config?: NcConfigFactory,
-    type?: string,
-  ): DbConfig {
-    const url = new URL(urlString);
-
-    let dbConfig: DbConfig;
-
-    if (url.protocol.startsWith('sqlite3')) {
-      dbConfig = {
-        client: 'sqlite3',
-        connection: {
-          client: 'sqlite3',
-          connection: {
-            filename:
-              url.searchParams.get('d') || url.searchParams.get('database'),
-          },
-          database:
-            url.searchParams.get('d') || url.searchParams.get('database'),
-        },
-      } as any;
-    } else {
-      const parsedQuery = {};
-      for (const [key, value] of url.searchParams.entries()) {
-        const fnd = knownQueryParams.find(
-          (param) => param.parameter === key || param.aliases.includes(key),
-        );
-        if (fnd) {
-          parsedQuery[fnd.parameter] = value;
-        } else {
-          parsedQuery[key] = value;
-        }
-      }
-
-      dbConfig = {
-        client: url.protocol.replace(':', ''),
-        connection: {
-          ...defaultConnectionConfig,
-          ...parsedQuery,
-          host: url.hostname,
-          port: +url.port,
-        },
-        // pool: {
-        //   min: 1,
-        //   max: 1
-        // },
-        acquireConnectionTimeout: 600000,
-      } as any;
-
-      if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
-        dbConfig.connection.ssl = true;
-      }
-
-      if (
-        url.searchParams.get('keyFilePath') &&
-        url.searchParams.get('certFilePath') &&
-        url.searchParams.get('caFilePath')
-      ) {
-        dbConfig.connection.ssl = {
-          keyFilePath: url.searchParams.get('keyFilePath'),
-          certFilePath: url.searchParams.get('certFilePath'),
-          caFilePath: url.searchParams.get('caFilePath'),
-        };
-      }
-    }
-
-    if (config && !config.title) {
-      config.title =
-        url.searchParams.get('t') ||
-        url.searchParams.get('title') ||
-        this.generateRandomTitle();
-    }
-
-    Object.assign(dbConfig, {
-      meta: {
-        tn: 'nc_evolutions',
-        allSchemas:
-          !!url.searchParams.get('allSchemas') ||
-          !(url.searchParams.get('d') || url.searchParams.get('database')),
-        api: {
-          prefix: url.searchParams.get('apiPrefix') || '',
-          swagger: true,
-          type:
-            type ||
-            ((url.searchParams.get('api') ||
-              url.searchParams.get('a')) as any) ||
-            'rest',
-        },
-        dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
-        metaTables: 'db',
-        migrations: {
-          disabled: false,
-          name: 'nc_evolutions',
-        },
-      },
-    });
-
-    return dbConfig;
-  }
-
-  private static generateRandomTitle(): string {
-    return ''; /*uniqueNamesGenerator({
-      dictionaries: [[starWars], [adjectives, animals]][
-        Math.floor(Math.random() * 2)
-      ],
-    })
-      .toLowerCase()
-      .replace(/[ -]/g, '_');*/
-  }
-
-  static async metaUrlToDbConfig(urlString) {
-    const url = new URL(urlString);
-
-    let dbConfig;
-
-    if (url.protocol.startsWith('sqlite3')) {
-      const db = url.searchParams.get('d') || url.searchParams.get('database');
-      dbConfig = {
-        client: 'sqlite3',
-        connection: {
-          filename: db,
-        },
-        ...(db === ':memory:'
-          ? {
-              pool: {
-                min: 1,
-                max: 1,
-                // disposeTimeout: 360000*1000,
-                idleTimeoutMillis: 360000 * 1000,
-              },
-            }
-          : {}),
-      };
-    } else {
-      const parsedQuery = {};
-      for (const [key, value] of url.searchParams.entries()) {
-        const fnd = knownQueryParams.find(
-          (param) => param.parameter === key || param.aliases.includes(key),
-        );
-        if (fnd) {
-          parsedQuery[fnd.parameter] = value;
-        } else {
-          parsedQuery[key] = value;
-        }
-      }
-
-      dbConfig = {
-        client: url.protocol.replace(':', ''),
-        connection: {
-          ...defaultConnectionConfig,
-          ...parsedQuery,
-          host: url.hostname,
-          port: +url.port,
-        },
-        acquireConnectionTimeout: 600000,
-        ...(url.searchParams.has('search_path')
-          ? {
-              searchPath: url.searchParams.get('search_path').split(','),
-            }
-          : {}),
-      };
-      if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
-        dbConfig.connection.ssl = true;
-      }
-    }
-    url.searchParams.forEach((_value, key) => {
-      let value: any = _value;
-      if (value === 'true') {
-        value = true;
-      } else if (value === 'false') {
-        value = false;
-      } else if (/^\d+$/.test(value)) {
-        value = +value;
-      }
-      // todo: implement config read from JSON file or JSON env val read
-      if (
-        ![
-          'password',
-          'p',
-          'database',
-          'd',
-          'user',
-          'u',
-          'search_path',
-        ].includes(key)
-      ) {
-        key.split('.').reduce((obj, k, i, arr) => {
-          return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
-        }, dbConfig);
-      }
-    });
-
-    if (
-      dbConfig?.connection?.ssl &&
-      typeof dbConfig?.connection?.ssl === 'object'
-    ) {
-      if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
-        dbConfig.connection.ssl.ca = (
-          await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
-        ).toString();
-        delete dbConfig.connection.ssl.caFilePath;
-      }
-      if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
-        dbConfig.connection.ssl.key = (
-          await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
-        ).toString();
-        delete dbConfig.connection.ssl.keyFilePath;
-      }
-      if (
-        dbConfig.connection.ssl.certFilePath &&
-        !dbConfig.connection.ssl.cert
-      ) {
-        dbConfig.connection.ssl.cert = (
-          await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
-        ).toString();
-        delete dbConfig.connection.ssl.certFilePath;
-      }
-    }
-
-    return dbConfig;
-  }
-
-  public static async makeProjectConfigFromUrl(
-    url,
-    type?: string,
-  ): Promise<NcConfig> {
-    const config = new NcConfigFactory();
-    const dbConfig = this.urlToDbConfig(url, '', config, type);
-    // config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
-    config.envs['_noco'].db.push(dbConfig);
-
-    if (process.env.NC_AUTH_ADMIN_SECRET) {
-      config.auth = {
-        masterKey: {
-          secret: process.env.NC_AUTH_ADMIN_SECRET,
-        },
-      };
-    } else if (process.env.NC_NO_AUTH) {
-      config.auth = {
-        disabled: true,
-      };
-      // } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
-    } else if (config?.envs?.['_noco']?.db?.[0]) {
-      config.auth = {
-        jwt: {
-          // dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
-          dbAlias:
-            process.env.NC_AUTH_JWT_DB_ALIAS ||
-            config.envs['_noco'].db[0].meta.dbAlias,
-          secret: process.env.NC_AUTH_JWT_SECRET,
-        },
-      };
-    }
-
-    if (process.env.NC_DB) {
-      config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
-    }
-
-    if (process.env.NC_TRY) {
-      config.try = true;
-      config.meta.db = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        pool: {
-          min: 1,
-          max: 1,
-          // disposeTimeout: 360000*1000,
-          idleTimeoutMillis: 360000 * 1000,
-        },
-      } as any;
-    }
-
-    if (process.env.NC_MAILER) {
-      config.mailer = {
-        from: process.env.NC_MAILER_FROM,
-        options: {
-          host: process.env.NC_MAILER_HOST,
-          port: parseInt(process.env.NC_MAILER_PORT, 10),
-          secure: process.env.NC_MAILER_SECURE === 'true',
-          auth: {
-            user: process.env.NC_MAILER_USER,
-            pass: process.env.NC_MAILER_PASS,
-          },
-        },
-      };
-    }
-
-    if (process.env.NC_PUBLIC_URL) {
-      // config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.publicUrl = process.env.NC_PUBLIC_URL;
-    }
-
-    config.port = +(process?.env?.PORT ?? 8080);
-    // config.env = process.env?.NODE_ENV || 'dev';
-    // config.workingEnv = process.env?.NODE_ENV || 'dev';
-    config.env = '_noco';
-    config.workingEnv = '_noco';
-    config.toolDir = this.getToolDir();
-    config.projectType =
-      type ||
-      config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
-      'rest';
-
-    return config;
-  }
-
-  public static async makeProjectConfigFromConnection(
-    dbConnectionConfig: any,
-    type?: string,
-  ): Promise<NcConfig> {
-    const config = new NcConfigFactory();
-    let dbConfig = dbConnectionConfig;
-
-    if (dbConfig.client === 'sqlite3') {
-      dbConfig = {
-        client: 'sqlite3',
-        connection: {
-          ...dbConnectionConfig,
-          database: dbConnectionConfig.connection.filename,
-        },
-      };
-    }
-
-    // todo:
-    const key = '';
-    Object.assign(dbConfig, {
-      meta: {
-        tn: 'nc_evolutions',
-        api: {
-          prefix: '',
-          swagger: true,
-          type: type || 'rest',
-        },
-        dbAlias: `db${key}`,
-        metaTables: 'db',
-        migrations: {
-          disabled: false,
-          name: 'nc_evolutions',
-        },
-      },
-    });
-
-    // config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
-    config.envs['_noco'].db.push(dbConfig);
-
-    if (process.env.NC_AUTH_ADMIN_SECRET) {
-      config.auth = {
-        masterKey: {
-          secret: process.env.NC_AUTH_ADMIN_SECRET,
-        },
-      };
-    } else if (process.env.NC_NO_AUTH) {
-      config.auth = {
-        disabled: true,
-      };
-      // } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
-    } else if (config?.envs?.['_noco']?.db?.[0]) {
-      config.auth = {
-        jwt: {
-          // dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
-          dbAlias:
-            process.env.NC_AUTH_JWT_DB_ALIAS ||
-            config.envs['_noco'].db[0].meta.dbAlias,
-          secret: process.env.NC_AUTH_JWT_SECRET,
-        },
-      };
-    }
-
-    if (process.env.NC_DB) {
-      config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
-    }
-
-    if (process.env.NC_TRY) {
-      config.try = true;
-      config.meta.db = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        pool: {
-          min: 1,
-          max: 1,
-          // disposeTimeout: 360000*1000,
-          idleTimeoutMillis: 360000 * 1000,
-        },
-      } as any;
-    }
-
-    if (process.env.NC_PUBLIC_URL) {
-      // config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.publicUrl = process.env.NC_PUBLIC_URL;
-    }
-
-    config.port = +(process?.env?.PORT ?? 8080);
-    // config.env = process.env?.NODE_ENV || 'dev';
-    // config.workingEnv = process.env?.NODE_ENV || 'dev';
-    config.env = '_noco';
-    config.workingEnv = '_noco';
-    config.toolDir = process.env.NC_TOOL_DIR || process.cwd();
-    config.projectType =
-      type ||
-      config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
-      'rest';
-
-    return config;
-  }
-
-  public static async metaDbCreateIfNotExist(args: NcConfig) {
-    if (args.meta?.db?.client === 'sqlite3') {
-      const metaSqlClient = await SqlClientFactory.create({
-        ...args.meta.db,
-        connection: args.meta.db,
-      });
-      await metaSqlClient.createDatabaseIfNotExists({
-        database: args.meta.db?.connection?.filename,
-      });
-    } else {
-      const metaSqlClient = await SqlClientFactory.create(args.meta.db);
-      await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection);
-      await metaSqlClient.knex.destroy();
-    }
-
-    /*    const dbPath = path.join(args.toolDir, 'xc.db')
-        const exists = fs.existsSync(dbPath);
-        if (!exists) {
-          const fd = fs.openSync(dbPath, "w");
-          fs.closeSync(fd);
-        }
-        */
-  }
-
-  public version = '0.6';
-  public port: number;
-  public auth?: any;
-  public env: 'production' | 'dev' | 'test' | string;
-  public workingEnv: string;
-  public toolDir: string;
-  public envs: {
-    [p: string]: { db: DbConfig[]; api?: any; publicUrl?: string };
-  };
-  // public projectType: "rest" | "graphql" | "grpc";
-  public queriesFolder: string | string[] = '';
-  public seedsFolder: string | string[];
-  public title: string;
-  public publicUrl: string;
-  public projectType;
-  public meta = {
-    db: {
-      client: 'sqlite3',
-      connection: {
-        filename: 'noco.db',
-      },
-    },
-  };
-  public mailer: any;
-  public try = false;
-
-  public dashboardPath = '/dashboard';
-
-  constructor() {
-    this.envs = { _noco: { db: [] } };
-  }
-
-  public static async jdbcToXcUrl() {
-    if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
-      const database_url = await promisify(fs.readFile)(
-        process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
-        'utf-8',
-      );
-      process.env.NC_DB = this.extractXcUrlFromJdbc(database_url);
-    } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
-      process.env.NC_DB = this.extractXcUrlFromJdbc(
-        process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
-      );
-    }
-  }
-
-  public static extractXcUrlFromJdbc(url: string, rtConfig = false) {
-    // drop the jdbc prefix
-    if (url.startsWith('jdbc:')) {
-      url = url.substring(5);
-    }
-
-    const config = parseDbUrl(url);
-
-    const parsedConfig: {
-      driver?: string;
-      host?: string;
-      port?: string;
-      database?: string;
-      user?: string;
-      password?: string;
-      ssl?: string;
-    } = {};
-    for (const [key, value] of Object.entries(config)) {
-      const fnd = knownQueryParams.find(
-        (param) => param.parameter === key || param.aliases.includes(key),
-      );
-      if (fnd) {
-        parsedConfig[fnd.parameter] = value;
-      } else {
-        parsedConfig[key] = value;
-      }
-    }
-
-    if (!parsedConfig?.port)
-      parsedConfig.port =
-        defaultClientPortMapping[
-          driverClientMapping[parsedConfig.driver] || parsedConfig.driver
-        ];
-
-    if (rtConfig) {
-      const { driver, ...connectionConfig } = parsedConfig;
-
-      const client = driverClientMapping[driver] || driver;
-
-      const avoidSSL = [
-        'localhost',
-        '127.0.0.1',
-        'host.docker.internal',
-        '172.17.0.1',
-      ];
-
-      if (
-        client === 'pg' &&
-        !connectionConfig?.ssl &&
-        !avoidSSL.includes(connectionConfig.host)
-      ) {
-        connectionConfig.ssl = 'true';
-      }
-
-      return {
-        client: client,
-        connection: {
-          ...connectionConfig,
-        },
-      } as any;
-    }
-
-    const { driver, host, port, database, user, password, ...extra } =
-      parsedConfig;
-
-    const extraParams = [];
-
-    for (const [key, value] of Object.entries(extra)) {
-      extraParams.push(`${key}=${value}`);
-    }
-
-    const res = `${driverClientMapping[driver] || driver}://${host}${
-      port ? `:${port}` : ''
-    }?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
-      database ? `d=${database}&` : ''
-    }${extraParams.join('&')}`;
-
-    return res;
-  }
-
-  // public static initOneClickDeployment() {
-  //   if (process.env.NC_ONE_CLICK) {
-  //     const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL);
-  //     process.env.NC_DB = url;
-  //   }
-  // }
-}
-
-export { defaultConnectionConfig, defaultConnectionOptions };

packages/nocodb/src/utils/common/NcConnectionMgr.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import SqlClientFactory from '../../db/sql-client/lib/SqlClientFactory';
 import { XKnex } from '../../db/CustomKnex';
-import { defaultConnectionConfig } from '../NcConfigFactory';
+import { defaultConnectionConfig } from '../nc-config';
 // import type { NcConfig } from '../../../interface/config';
 import type { Knex } from 'knex';
 // import type NcMetaIO from '../../meta/NcMetaIO';

packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

@@ -3,7 +3,7 @@ import { XKnex } from '../../db/CustomKnex';
 import {
   defaultConnectionConfig,
   defaultConnectionOptions,
-} from '../NcConfigFactory';
+} from '../nc-config';
 import Noco from '../../Noco';
 import type Base from '../../models/Base';
 

packages/nocodb/src/utils/nc-config/NcConfig.ts

@@ -0,0 +1,182 @@
+import * as path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory';
+import { getToolDir, metaUrlToDbConfig } from './helpers';
+import { DriverClient } from './interfaces';
+import type { DbConfig } from './interfaces';
+
+export class NcConfig {
+  version: string;
+  meta: {
+    db: DbConfig;
+  } = {
+    db: {
+      client: DriverClient.SQLITE,
+      connection: {
+        filename: 'noco.db',
+      },
+    },
+  };
+  auth: {
+    jwt: {
+      secret: string;
+      options?: any;
+    };
+  };
+
+  // if this is true, port is not exposed
+  worker: boolean;
+
+  toolDir: string;
+
+  // exposed instance port
+  port: number;
+
+  // if this is true, use sqlite3 :memory: as meta db
+  try: boolean;
+
+  // optional
+  publicUrl?: string;
+  dashboardPath?: string;
+
+  // TODO what is this?
+  envs: any;
+
+  queriesFolder: string;
+  env: string;
+  workingEnv: string;
+  projectType: string;
+
+  private constructor() {}
+
+  public static async create(param: {
+    meta: {
+      metaUrl?: string;
+      metaJson?: string;
+      metaJsonFile?: string;
+    };
+    secret?: string;
+    port?: string | number;
+    tryMode?: boolean;
+    worker?: boolean;
+    dashboardPath?: string;
+    publicUrl?: string;
+  }): Promise<NcConfig> {
+    const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } =
+      param;
+
+    const ncConfig = new NcConfig();
+
+    ncConfig.auth = {
+      jwt: {
+        secret: secret,
+      },
+    };
+
+    ncConfig.port = +(port ?? 8080);
+    ncConfig.toolDir = getToolDir();
+    ncConfig.worker = worker ?? false;
+
+    ncConfig.env = '_noco';
+    ncConfig.workingEnv = '_noco';
+
+    ncConfig.projectType =
+      ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
+
+    if (ncConfig.meta?.db?.connection?.filename) {
+      ncConfig.meta.db.connection.filename = path.join(
+        ncConfig.toolDir,
+        ncConfig.meta.db.connection.filename,
+      );
+    }
+
+    if (tryMode) {
+      ncConfig.try = true;
+      ncConfig.meta.db = {
+        client: DriverClient.SQLITE,
+        connection: ':memory:' as any,
+        pool: {
+          min: 1,
+          max: 1,
+          // disposeTimeout: 360000*1000,
+          idleTimeoutMillis: 360000 * 1000,
+        },
+      };
+    } else {
+      if (meta?.metaUrl) {
+        ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl);
+      } else if (meta?.metaJson) {
+        ncConfig.meta.db = JSON.parse(meta.metaJson);
+      } else if (meta?.metaJsonFile) {
+        if (!(await promisify(fs.exists)(meta.metaJsonFile))) {
+          throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`);
+        }
+        const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, {
+          encoding: 'utf8',
+        });
+        ncConfig.meta.db = JSON.parse(fileContent);
+      }
+    }
+
+    if (publicUrl) {
+      ncConfig.envs['_noco'].publicUrl = publicUrl;
+      ncConfig.publicUrl = publicUrl;
+    }
+
+    if (dashboardPath) {
+      ncConfig.dashboardPath = dashboardPath;
+    }
+
+    try {
+      // make sure meta db exists
+      await ncConfig.metaDbCreateIfNotExist();
+    } catch (e) {
+      throw new Error(e);
+    }
+
+    return ncConfig;
+  }
+
+  public static async createByEnv(): Promise<NcConfig> {
+    return NcConfig.create({
+      meta: {
+        metaUrl: process.env.NC_DB,
+        metaJson: process.env.NC_DB_JSON,
+        metaJsonFile: process.env.NC_DB_JSON_FILE,
+      },
+      secret: process.env.NC_AUTH_JWT_SECRET,
+      port: process.env.NC_PORT,
+      tryMode: !!process.env.NC_TRY,
+      worker: !!process.env.NC_WORKER,
+      dashboardPath: process.env.NC_DASHBOARD_PATH,
+      publicUrl: process.env.NC_PUBLIC_URL,
+    });
+  }
+
+  private async metaDbCreateIfNotExist() {
+    if (this.meta?.db?.client === 'sqlite3') {
+      const metaSqlClient = await SqlClientFactory.create({
+        ...this.meta.db,
+        connection: this.meta.db,
+      });
+      if (this.meta.db?.connection?.filename) {
+        await metaSqlClient.createDatabaseIfNotExists({
+          database: this.meta.db?.connection?.filename,
+        });
+      } else {
+        throw new Error('Configuration missing meta db connection');
+      }
+    } else {
+      const metaSqlClient = await SqlClientFactory.create(this.meta.db);
+      if (this.meta.db?.connection?.database) {
+        await metaSqlClient.createDatabaseIfNotExists(
+          (this.meta.db as any).connection,
+        );
+        await metaSqlClient.knex.destroy();
+      } else {
+        throw new Error('Configuration missing meta db connection');
+      }
+    }
+  }
+}

packages/nocodb/src/utils/nc-config/constants.ts

@@ -0,0 +1,84 @@
+export const driverClientMapping = {
+  mysql: 'mysql2',
+  mariadb: 'mysql2',
+  postgres: 'pg',
+  postgresql: 'pg',
+  sqlite: 'sqlite3',
+  mssql: 'mssql',
+};
+
+export const defaultClientPortMapping = {
+  mysql: 3306,
+  mysql2: 3306,
+  postgres: 5432,
+  pg: 5432,
+  mssql: 1433,
+};
+
+export const defaultConnectionConfig: any = {
+  // https://github.com/knex/knex/issues/97
+  // timezone: process.env.NC_TIMEZONE || 'UTC',
+  dateStrings: true,
+};
+
+// default knex options
+export const defaultConnectionOptions = {
+  pool: {
+    min: 0,
+    max: 10,
+  },
+};
+
+export const avoidSSL = [
+  'localhost',
+  '127.0.0.1',
+  'host.docker.internal',
+  '172.17.0.1',
+];
+
+export const knownQueryParams = [
+  {
+    parameter: 'database',
+    aliases: ['d', 'db'],
+  },
+  {
+    parameter: 'password',
+    aliases: ['p'],
+  },
+  {
+    parameter: 'user',
+    aliases: ['u'],
+  },
+  {
+    parameter: 'title',
+    aliases: ['t'],
+  },
+  {
+    parameter: 'keyFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'certFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'caFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'ssl',
+    aliases: [],
+  },
+  {
+    parameter: 'options',
+    aliases: ['opt', 'opts'],
+  },
+];
+
+export enum DriverClient {
+  MYSQL = 'mysql2',
+  MSSQL = 'mssql',
+  PG = 'pg',
+  SQLITE = 'sqlite3',
+  SNOWFLAKE = 'snowflake',
+}

packages/nocodb/src/utils/nc-config/helpers.ts

@@ -0,0 +1,324 @@
+import fs from 'fs';
+import { URL } from 'url';
+import { promisify } from 'util';
+import parseDbUrl from 'parse-database-url';
+import {
+  avoidSSL,
+  defaultClientPortMapping,
+  defaultConnectionConfig,
+  defaultConnectionOptions,
+  driverClientMapping,
+  knownQueryParams,
+} from './constants';
+import { DriverClient } from './interfaces';
+import type { Connection, DbConfig } from './interfaces';
+
+export async function prepareEnv() {
+  if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
+    const database_url = await promisify(fs.readFile)(
+      process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
+      'utf-8',
+    );
+    process.env.NC_DB = jdbcToXcUrl(database_url);
+  } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
+    process.env.NC_DB = jdbcToXcUrl(
+      process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
+    );
+  }
+}
+
+export function getToolDir() {
+  return process.env.NC_TOOL_DIR || process.cwd();
+}
+
+export function jdbcToXcConfig(url: string): DbConfig {
+  // drop the jdbc prefix
+  url.replace(/^jdbc:/, '');
+
+  const config = parseDbUrl(url);
+
+  const parsedConfig: Connection = {};
+
+  for (const [key, value] of Object.entries(config)) {
+    const fnd = knownQueryParams.find(
+      (param) => param.parameter === key || param.aliases.includes(key),
+    );
+    if (fnd) {
+      parsedConfig[fnd.parameter] = value;
+    } else {
+      parsedConfig[key] = value;
+    }
+  }
+
+  if (!parsedConfig?.port) {
+    parsedConfig.port =
+      defaultClientPortMapping[
+        driverClientMapping[parsedConfig.driver] || parsedConfig.driver
+      ];
+  }
+
+  const { driver, ...connectionConfig } = parsedConfig;
+
+  const client = driverClientMapping[driver] || driver;
+
+  if (
+    client === 'pg' &&
+    !connectionConfig?.ssl &&
+    !avoidSSL.includes(connectionConfig.host)
+  ) {
+    connectionConfig.ssl = true;
+  }
+
+  return {
+    client: client,
+    connection: {
+      ...connectionConfig,
+    },
+  } as DbConfig;
+}
+
+export function jdbcToXcUrl(url: string): string {
+  // drop the jdbc prefix
+  url.replace(/^jdbc:/, '');
+
+  const config = parseDbUrl(url);
+
+  const parsedConfig: Connection = {};
+
+  for (const [key, value] of Object.entries(config)) {
+    const fnd = knownQueryParams.find(
+      (param) => param.parameter === key || param.aliases.includes(key),
+    );
+    if (fnd) {
+      parsedConfig[fnd.parameter] = value;
+    } else {
+      parsedConfig[key] = value;
+    }
+  }
+
+  if (!parsedConfig?.port) {
+    parsedConfig.port =
+      defaultClientPortMapping[
+        driverClientMapping[parsedConfig.driver] || parsedConfig.driver
+      ];
+  }
+
+  const { driver, host, port, database, user, password, ...extra } =
+    parsedConfig;
+
+  const extraParams = [];
+
+  for (const [key, value] of Object.entries(extra)) {
+    extraParams.push(`${key}=${value}`);
+  }
+
+  const res = `${driverClientMapping[driver] || driver}://${host}${
+    port ? `:${port}` : ''
+  }?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
+    database ? `d=${database}&` : ''
+  }${extraParams.join('&')}`;
+
+  return res;
+}
+
+export function xcUrlToDbConfig(
+  urlString: string,
+  key = '',
+  type?: string,
+): DbConfig {
+  const url = new URL(urlString);
+
+  let dbConfig: DbConfig;
+
+  if (url.protocol.startsWith('sqlite3')) {
+    dbConfig = {
+      client: 'sqlite3',
+      connection: {
+        client: 'sqlite3',
+        connection: {
+          filename:
+            url.searchParams.get('d') || url.searchParams.get('database'),
+        },
+        database: url.searchParams.get('d') || url.searchParams.get('database'),
+      },
+    } as any;
+  } else {
+    const parsedQuery = {};
+    for (const [key, value] of url.searchParams.entries()) {
+      const fnd = knownQueryParams.find(
+        (param) => param.parameter === key || param.aliases.includes(key),
+      );
+      if (fnd) {
+        parsedQuery[fnd.parameter] = value;
+      } else {
+        parsedQuery[key] = value;
+      }
+    }
+
+    dbConfig = {
+      client: url.protocol.replace(':', '') as DriverClient,
+      connection: {
+        ...parsedQuery,
+        host: url.hostname,
+        port: +url.port,
+      },
+      acquireConnectionTimeout: 600000,
+    };
+
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
+      dbConfig.connection.ssl = true;
+    }
+
+    if (
+      url.searchParams.get('keyFilePath') &&
+      url.searchParams.get('certFilePath') &&
+      url.searchParams.get('caFilePath')
+    ) {
+      dbConfig.connection.ssl = {
+        keyFilePath: url.searchParams.get('keyFilePath'),
+        certFilePath: url.searchParams.get('certFilePath'),
+        caFilePath: url.searchParams.get('caFilePath'),
+      };
+    }
+  }
+
+  /* TODO check if this is needed
+    if (config && !config.title) {
+      config.title =
+        url.searchParams.get('t') ||
+        url.searchParams.get('title') ||
+        this.generateRandomTitle();
+    }
+  */
+
+  Object.assign(dbConfig, {
+    meta: {
+      tn: 'nc_evolutions',
+      allSchemas:
+        !!url.searchParams.get('allSchemas') ||
+        !(url.searchParams.get('d') || url.searchParams.get('database')),
+      api: {
+        prefix: url.searchParams.get('apiPrefix') || '',
+        swagger: true,
+        type:
+          type ||
+          ((url.searchParams.get('api') || url.searchParams.get('a')) as any) ||
+          'rest',
+      },
+      dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
+      metaTables: 'db',
+      migrations: {
+        disabled: false,
+        name: 'nc_evolutions',
+      },
+    },
+  });
+
+  return dbConfig;
+}
+
+export async function metaUrlToDbConfig(urlString): Promise<DbConfig> {
+  const url = new URL(urlString);
+
+  let dbConfig: DbConfig;
+
+  if (url.protocol.startsWith('sqlite3')) {
+    const db = url.searchParams.get('d') || url.searchParams.get('database');
+    dbConfig = {
+      client: DriverClient.SQLITE,
+      connection: {
+        filename: db,
+      },
+      ...(db === ':memory:'
+        ? {
+            pool: {
+              min: 1,
+              max: 1,
+              // disposeTimeout: 360000*1000,
+              idleTimeoutMillis: 360000 * 1000,
+            },
+          }
+        : {}),
+    };
+  } else {
+    const parsedQuery = {};
+    for (const [key, value] of url.searchParams.entries()) {
+      const fnd = knownQueryParams.find(
+        (param) => param.parameter === key || param.aliases.includes(key),
+      );
+      if (fnd) {
+        parsedQuery[fnd.parameter] = value;
+      } else {
+        parsedQuery[key] = value;
+      }
+    }
+
+    dbConfig = {
+      client: url.protocol.replace(':', '') as DriverClient,
+      connection: {
+        ...defaultConnectionConfig,
+        ...parsedQuery,
+        host: url.hostname,
+        port: +url.port,
+      },
+      acquireConnectionTimeout: 600000,
+      ...defaultConnectionOptions,
+      ...(url.searchParams.has('search_path')
+        ? {
+            searchPath: url.searchParams.get('search_path').split(','),
+          }
+        : {}),
+    };
+
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
+      dbConfig.connection.ssl = true;
+    }
+  }
+
+  url.searchParams.forEach((_value, key) => {
+    let value: any = _value;
+    if (value === 'true') {
+      value = true;
+    } else if (value === 'false') {
+      value = false;
+    } else if (/^\d+$/.test(value)) {
+      value = +value;
+    }
+    // todo: implement config read from JSON file or JSON env val read
+    if (
+      !['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes(
+        key,
+      )
+    ) {
+      key.split('.').reduce((obj, k, i, arr) => {
+        return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
+      }, dbConfig);
+    }
+  });
+
+  if (
+    dbConfig?.connection?.ssl &&
+    typeof dbConfig?.connection?.ssl === 'object'
+  ) {
+    if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
+      dbConfig.connection.ssl.ca = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.caFilePath;
+    }
+    if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
+      dbConfig.connection.ssl.key = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.keyFilePath;
+    }
+    if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) {
+      dbConfig.connection.ssl.cert = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.certFilePath;
+    }
+  }
+
+  return dbConfig;
+}

packages/nocodb/src/utils/nc-config/index.ts

@@ -0,0 +1,4 @@
+export * from './helpers';
+export * from './interfaces';
+export * from './constants';
+export * from './NcConfig';

packages/nocodb/src/utils/nc-config/interfaces.ts

@@ -0,0 +1,39 @@
+import { DriverClient } from './constants';
+
+interface Connection {
+  driver?: DriverClient;
+  host?: string;
+  port?: number;
+  database?: string;
+  user?: string;
+  password?: string;
+  ssl?:
+    | boolean
+    | {
+        ca?: string;
+        cert?: string;
+        key?: string;
+        caFilePath?: string;
+        certFilePath?: string;
+        keyFilePath?: string;
+      };
+  filename?: string;
+}
+
+interface DbConfig {
+  client: DriverClient;
+  connection: Connection;
+  acquireConnectionTimeout?: number;
+  useNullAsDefault?: boolean;
+  pool?: {
+    min?: number;
+    max?: number;
+    idleTimeoutMillis?: number;
+  };
+  migrations?: {
+    directory?: string;
+    tableName?: string;
+  };
+}
+
+export { DriverClient, Connection, DbConfig };

packages/nocodb/tests/unit/TestDbMngr.ts

@@ -2,9 +2,9 @@ import fs from 'fs';
 import process from 'process';
 import { knex } from 'knex';
 import SqlMgrv2 from '../../src/db/sql-mgr/v2/SqlMgrv2';
+import { jdbcToXcUrl, xcUrlToDbConfig } from '../../src/utils/nc-config';
 import type { Knex } from 'knex';
 import type { DbConfig } from '../../src/interface/config';
-import NcConfigFactory from '../../src/utils/NcConfigFactory'
 
 export default class TestDbMngr {
   public static readonly dbName = 'test_meta';
@@ -75,7 +75,7 @@ export default class TestDbMngr {
 
   private static async isDbConfigured() {
     const { user, password, host, port, client } = TestDbMngr.connection;
-    const config = NcConfigFactory.urlToDbConfig(
+    const config = xcUrlToDbConfig(
       `${client}://${user}:${password}@${host}:${port}`,
     );
     config.connection = {
@@ -84,7 +84,7 @@ export default class TestDbMngr {
       host,
       port,
     };
-    const result = await TestDbMngr.testConnection(config);
+    const result = await TestDbMngr.testConnection(config as any);
     return result.code !== -1;
   }
   static async connectDb() {
@@ -95,9 +95,9 @@ export default class TestDbMngr {
       ] = `${client}://${user}:${password}@${host}:${port}/${TestDbMngr.dbName}`;
     }
 
-    TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig(
+    TestDbMngr.dbConfig = xcUrlToDbConfig(
-      NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`]),
+      jdbcToXcUrl(process.env[`DATABASE_URL`]),
-    );
+    ) as any;
     this.dbConfig.meta = {
       tn: 'nc_evolutions',
       dbAlias: 'db',

packages/nocodb/tests/unit/model/tests/baseModelSql.test.ts

@@ -1,20 +1,20 @@
 import 'mocha';
-import { BaseModelSqlv2 } from '../../../../src/db/BaseModelSqlv2'
+import { expect } from 'chai';
-import NcConnectionMgrv2 from '../../../../src/utils/common/NcConnectionMgrv2'
+import { BaseModelSqlv2 } from '../../../../src/db/BaseModelSqlv2';
+import NcConnectionMgrv2 from '../../../../src/utils/common/NcConnectionMgrv2';
 import init from '../../init';
 import { createProject } from '../../factory/project';
 import { createTable } from '../../factory/table';
 import Base from '../../../../src/models/Base';
-import Model from '../../../../src/models/Model';
-import Project from '../../../../src/models/Project';
-import View from '../../../../src/models/View';
 import { createRow, generateDefaultRowAttributes } from '../../factory/row';
 import Audit from '../../../../src/models/Audit';
-import { expect } from 'chai';
 import Filter from '../../../../src/models/Filter';
 import { createLtarColumn } from '../../factory/column';
-import LinkToAnotherRecordColumn from '../../../../src/models/LinkToAnotherRecordColumn';
 import { isPg, isSqlite } from '../../init/db';
+import type View from '../../../../src/models/View';
+import type Project from '../../../../src/models/Project';
+import type Model from '../../../../src/models/Model';
+import type LinkToAnotherRecordColumn from '../../../../src/models/LinkToAnotherRecordColumn';
 
 function baseModelSqlTests() {
   let context;
@@ -44,11 +44,11 @@ function baseModelSqlTests() {
     };
     const columns = await table.getColumns();
 
-    let inputData: any = generateDefaultRowAttributes({ columns });
+    const inputData: any = generateDefaultRowAttributes({ columns });
     const response = await baseModelSql.insert(
       generateDefaultRowAttributes({ columns }),
       undefined,
-      request
+      request,
     );
     const insertedRow = (await baseModelSql.list())[0];
 
@@ -61,6 +61,10 @@ function baseModelSqlTests() {
 
       response.CreatedAt = new Date(response.CreatedAt).toISOString();
       response.UpdatedAt = new Date(response.UpdatedAt).toISOString();
+    } else if (isSqlite(context)) {
+      // append +00:00 to the date string
+      inputData.CreatedAt = `${inputData.CreatedAt}+00:00`;
+      inputData.UpdatedAt = `${inputData.UpdatedAt}+00:00`;
     }
 
     expect(insertedRow).to.include(inputData);
@@ -106,6 +110,10 @@ function baseModelSqlTests() {
       if (isPg(context)) {
         inputData.CreatedAt = new Date(inputData.CreatedAt).toISOString();
         inputData.UpdatedAt = new Date(inputData.UpdatedAt).toISOString();
+      } else if (isSqlite(context)) {
+        // append +00:00 to the date string
+        inputData.CreatedAt = `${inputData.CreatedAt}+00:00`;
+        inputData.UpdatedAt = `${inputData.UpdatedAt}+00:00`;
       }
       expect(insertedRows[index]).to.include(inputData);
     });
@@ -145,7 +153,7 @@ function baseModelSqlTests() {
     expect(updatedRow).to.include({ Id: rowId, Title: 'test' });
 
     const rowUpdatedAudit = (await Audit.projectAuditList(project.id, {})).find(
-      (audit) => audit.op_sub_type === 'UPDATE'
+      (audit) => audit.op_sub_type === 'UPDATE',
     );
     expect(rowUpdatedAudit).to.include({
       user: 'test@example.com',
@@ -156,7 +164,8 @@ function baseModelSqlTests() {
       row_id: '1',
       op_type: 'DATA',
       op_sub_type: 'UPDATE',
-      description: 'Record with ID 1 has been updated in Table Table1_Title.\nColumn "Title" got changed from "test-0" to "test"',
+      description:
+        'Record with ID 1 has been updated in Table Table1_Title.\nColumn "Title" got changed from "test-0" to "test"',
     });
   });
 
@@ -178,7 +187,7 @@ function baseModelSqlTests() {
 
     await baseModelSql.bulkUpdate(
       insertedRows.map((row) => ({ ...row, Title: `new-${row['Title']}` })),
-      { cookie: request }
+      { cookie: request },
     );
 
     const updatedRows = await baseModelSql.list();
@@ -229,7 +238,7 @@ function baseModelSqlTests() {
         ],
       },
       { Title: 'new-1' },
-      { cookie: request }
+      { cookie: request },
     );
 
     const updatedRows = await baseModelSql.list();
@@ -277,7 +286,7 @@ function baseModelSqlTests() {
 
     console.log('Delete record', await Audit.projectAuditList(project.id, {}));
     const rowDeletedAudit = (await Audit.projectAuditList(project.id, {})).find(
-      (audit) => audit.op_sub_type === 'DELETE'
+      (audit) => audit.op_sub_type === 'DELETE',
     );
     expect(rowDeletedAudit).to.include({
       user: 'test@example.com',
@@ -309,7 +318,7 @@ function baseModelSqlTests() {
       insertedRows
         .filter((row) => row['Id'] < 5)
         .map((row) => ({ id: row['Id'] })),
-      { cookie: request }
+      { cookie: request },
     );
 
     const remainingRows = await baseModelSql.list();
@@ -359,7 +368,7 @@ function baseModelSqlTests() {
           }),
         ],
       },
-      { cookie: request }
+      { cookie: request },
     );
 
     const remainingRows = await baseModelSql.list();
@@ -414,7 +423,7 @@ function baseModelSqlTests() {
         [ltarColumn.title]: [{ Id: childRow['Id'] }],
       },
       undefined,
-      request
+      request,
     );
 
     const childBaseModel = new BaseModelSqlv2({
@@ -470,7 +479,7 @@ function baseModelSqlTests() {
     await baseModelSql.insert(
       generateDefaultRowAttributes({ columns }),
       undefined,
-      request
+      request,
     );
     const insertedRow = await baseModelSql.readByPk(1);
 
@@ -487,7 +496,7 @@ function baseModelSqlTests() {
       view,
     });
     const updatedChildRow = await childBaseModel.readByPk(
-      insertedChildRow['Id']
+      insertedChildRow['Id'],
     );
 
     expect(updatedChildRow[childCol.column_name]).to.equal(insertedRow['Id']);
@@ -538,7 +547,7 @@ function baseModelSqlTests() {
     await baseModelSql.insert(
       generateDefaultRowAttributes({ columns }),
       undefined,
-      request
+      request,
     );
     const insertedRow = await baseModelSql.readByPk(1);
 
@@ -562,7 +571,7 @@ function baseModelSqlTests() {
       view,
     });
     const updatedChildRow = await childBaseModel.readByPk(
-      insertedChildRow['Id']
+      insertedChildRow['Id'],
     );
 
     expect(updatedChildRow[childCol.column_name]).to.be.null;

packages/nocodb/tests/unit/rest/tests/tableRow.test.ts

@@ -878,7 +878,9 @@ function tableTest() {
     }
   });
 
-  it('Sorted Formula column on rollup customer table', async function () {
+  // rollup usage in formula is currently not supported
+  // work in progress
+  it.skip('Sorted Formula column on rollup customer table', async function () {
     const rollupColumnTitle = 'Number of rentals';
     const rollupColumn = await createRollupColumn(context, {
       project: sakilaProject,

tests/playwright/package.json

@@ -11,6 +11,7 @@
     "test:repeat": "TRACE=true npx playwright test --workers=4 --repeat-each=12",
     "test:quick": "TRACE=true PW_QUICK_TEST=1 npx playwright  test --workers=4",
     "test:debug": "./startPlayWrightServer.sh; PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console npx playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 0 --workers 1 --max-failures=1",
+    "test:debug:intelliJ": "TRACE=true PWDEBUG=console npx playwright test --trace on -c playwright.config.ts --headed --project=chromium --retries 0 --workers 1 --max-failures=1",
     "test:debug:watch": "npx nodemon -e ts -w ./ -x \"npm run test:debug\"",
     "test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console npx playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1",
     "ci:test": "npx playwright test --workers=2",
@@ -46,7 +47,9 @@
     "body-parser": "^1.20.1",
     "dayjs": "^1.11.7",
     "express": "^4.18.2",
+    "knex": "^2.4.2",
     "nocodb-sdk": "file:../../packages/nocodb-sdk",
+    "sqlite3": "^5.1.6",
     "xlsx": "^0.18.5"
   }
 }

tests/playwright/pages/Dashboard/ExpandedForm/index.ts

@@ -1,6 +1,7 @@
 import { expect, Locator } from '@playwright/test';
 import BasePage from '../../Base';
 import { DashboardPage } from '..';
+import { DateTimeCellPageObject } from '../common/Cell/DateTimeCell';
 
 export class ExpandedFormPage extends BasePage {
   readonly dashboard: DashboardPage;
@@ -93,6 +94,14 @@ export class ExpandedFormPage extends BasePage {
         await field.locator(`[data-testid="nc-child-list-button-link-to"]`).click();
         await this.dashboard.linkRecord.select(value);
         break;
+      case 'dateTime':
+        await field.locator('.nc-cell').click();
+        // eslint-disable-next-line no-case-declarations
+        const dateTimeObj = new DateTimeCellPageObject(this.dashboard.grid.cell);
+        await dateTimeObj.selectDate({ date: value.slice(0, 10) });
+        await dateTimeObj.selectTime({ hour: +value.slice(11, 13), minute: +value.slice(14, 16) });
+        await dateTimeObj.save();
+        break;
     }
   }
 

tests/playwright/pages/Dashboard/TreeView.ts

@@ -48,6 +48,24 @@ export class TreeViewPage extends BasePage {
     await this.get().locator(`.nc-project-tree-tbl-${title}`).focus();
   }
 
+  async openBase({ title }: { title: string }) {
+    const nodes = await this.get().locator(`.ant-collapse`);
+    // loop through nodes.count() to find the node with title
+    for (let i = 0; i < (await nodes.count()); i++) {
+      const node = nodes.nth(i);
+      const nodeTitle = await node.innerText();
+      // check if nodeTitle contains title
+      if (nodeTitle.includes(title)) {
+        // click on node
+        await node.waitFor({ state: 'visible' });
+        await node.click();
+        break;
+      }
+    }
+
+    await this.rootPage.waitForTimeout(2000);
+  }
+
   // assumption: first view rendered is always GRID
   //
   async openTable({
@@ -72,6 +90,8 @@ export class TreeViewPage extends BasePage {
       }
     }
 
+    await this.get().locator(`.nc-project-tree-tbl-${title}`).waitFor({ state: 'visible' });
+
     if (networkResponse === true) {
       await this.waitForResponse({
         uiAction: () => this.get().locator(`.nc-project-tree-tbl-${title}`).click(),
@@ -86,7 +106,7 @@ export class TreeViewPage extends BasePage {
     }
   }
 
-  async createTable({ title, skipOpeningModal }: { title: string; skipOpeningModal?: boolean }) {
+  async createTable({ title, skipOpeningModal, mode }: { title: string; skipOpeningModal?: boolean; mode?: string }) {
     if (!skipOpeningModal) await this.get().locator('.nc-add-new-table').click();
 
     await this.dashboard.get().locator('.nc-modal-table-create').locator('.ant-modal-body').waitFor();
@@ -101,7 +121,7 @@ export class TreeViewPage extends BasePage {
     });
 
     // Tab render is slow for playwright
-    await this.dashboard.waitForTabRender({ title });
+    await this.dashboard.waitForTabRender({ title, mode });
   }
 
   async verifyTable({ title, index, exists = true }: { title: string; index?: number; exists?: boolean }) {
@@ -196,7 +216,6 @@ export class TreeViewPage extends BasePage {
       uiAction: () => this.rootPage.getByRole('button', { name: 'Confirm' }).click(),
       httpMethodsToMatch: ['POST'],
       requestUrlPathToMatch: `/api/v1/db/meta/duplicate/`,
-      responseJsonMatcher: json => json.name === 'duplicate-model',
     });
     await this.get().locator(`[data-testid="tree-view-table-${title} copy"]`).waitFor();
   }

tests/playwright/pages/Dashboard/common/Cell/DateTimeCell.ts

@@ -23,7 +23,7 @@ export class DateTimeCellPageObject extends BasePage {
   }
 
   async save() {
-    await this.rootPage.locator('button:has-text("Ok")').click();
+    await this.rootPage.locator('button:has-text("Ok"):visible').click();
   }
 
   async selectDate({
@@ -36,15 +36,15 @@ export class DateTimeCellPageObject extends BasePage {
     const [year, month, day] = date.split('-');
 
     // configure year
-    await this.rootPage.locator('.ant-picker-year-btn').click();
+    await this.rootPage.locator('.ant-picker-year-btn:visible').click();
     await this.rootPage.locator(`td[title="${year}"]`).click();
 
     // configure month
-    await this.rootPage.locator('.ant-picker-month-btn').click();
+    await this.rootPage.locator('.ant-picker-month-btn:visible').click();
     await this.rootPage.locator(`td[title="${year}-${month}"]`).click();
 
     // configure day
-    await this.rootPage.locator(`td[title="${year}-${month}-${day}"]`).click();
+    await this.rootPage.locator(`td[title="${year}-${month}-${day}"]:visible`).click();
   }
 
   async selectTime({
@@ -60,14 +60,20 @@ export class DateTimeCellPageObject extends BasePage {
     second?: number | null;
   }) {
     await this.rootPage
-      .locator(`.ant-picker-time-panel-column:nth-child(1) > .ant-picker-time-panel-cell:nth-child(${hour + 1})`)
+      .locator(
+        `.ant-picker-time-panel-column:nth-child(1) > .ant-picker-time-panel-cell:nth-child(${hour + 1}):visible`
+      )
       .click();
     await this.rootPage
-      .locator(`.ant-picker-time-panel-column:nth-child(2) > .ant-picker-time-panel-cell:nth-child(${minute + 1})`)
+      .locator(
+        `.ant-picker-time-panel-column:nth-child(2) > .ant-picker-time-panel-cell:nth-child(${minute + 1}):visible`
+      )
       .click();
     if (second != null) {
       await this.rootPage
-        .locator(`.ant-picker-time-panel-column:nth-child(3) > .ant-picker-time-panel-cell:nth-child(${second + 1})`)
+        .locator(
+          `.ant-picker-time-panel-column:nth-child(3) > .ant-picker-time-panel-cell:nth-child(${second + 1}):visible`
+        )
         .click();
     }
   }
@@ -75,4 +81,13 @@ export class DateTimeCellPageObject extends BasePage {
   async close() {
     await this.rootPage.keyboard.press('Escape');
   }
+
+  async setDateTime({ index, columnHeader, dateTime }: { index: number; columnHeader: string; dateTime: string }) {
+    const [date, time] = dateTime.split(' ');
+    const [hour, minute, second] = time.split(':');
+    await this.open({ index, columnHeader });
+    await this.selectDate({ date });
+    await this.selectTime({ hour: +hour, minute: +minute });
+    await this.save();
+  }
 }

tests/playwright/pages/Dashboard/common/Cell/index.ts

@@ -114,6 +114,11 @@ export class CellPageObject extends BasePage {
       // if text is found, return
       // if text is not found, throw error
       let count = 0;
+
+      if (!(this.parent instanceof SharedFormPage)) {
+        await this.rootPage.locator(`td[data-testid="cell-${columnHeader}-${index}"]`).waitFor({ state: 'visible' });
+      }
+
       await this.get({
         index,
         columnHeader,
@@ -360,4 +365,12 @@ export class CellPageObject extends BasePage {
     await this.get({ index, columnHeader }).press((await this.isMacOs()) ? 'Meta+C' : 'Control+C');
     await this.verifyToast({ message: 'Copied to clipboard' });
   }
+
+  async pasteFromClipboard({ index, columnHeader }: CellProps, ...clickOptions: Parameters<Locator['click']>) {
+    await this.get({ index, columnHeader }).scrollIntoViewIfNeeded();
+    await this.get({ index, columnHeader }).click(...clickOptions);
+    await (await this.get({ index, columnHeader }).elementHandle()).waitForElementState('stable');
+
+    await this.get({ index, columnHeader }).press((await this.isMacOs()) ? 'Meta+V' : 'Control+V');
+  }
 }

tests/playwright/pages/Dashboard/common/Toolbar/Filter.ts

@@ -298,4 +298,4 @@ export class ToolbarFilterPage extends BasePage {
 
     return opListText;
   }
-}
+}

tests/playwright/tests/db/viewGridShare.spec.ts

@@ -83,7 +83,7 @@ test.describe('Shared view', () => {
     }
 
     const expectedRecordsByDb = isSqlite(context) || isPg(context) ? sqliteExpectedRecords : expectedRecords;
-    await new Promise((resolve) => setTimeout(resolve, 1000));
+    await new Promise(resolve => setTimeout(resolve, 1000));
     // verify order of records (original sort & filter)
     for (const record of expectedRecordsByDb) {
       await sharedPage.grid.cell.verify(record);

tests/playwright/tests/utils/config.ts

@@ -0,0 +1,48 @@
+const knexConfig = {
+  pg: {
+    client: 'pg',
+    connection: {
+      host: 'localhost',
+      port: 5432,
+      user: 'postgres',
+      password: 'password',
+      database: 'postgres',
+      multipleStatements: true,
+    },
+    searchPath: ['public', 'information_schema'],
+    pool: { min: 0, max: 5 },
+  },
+  mysql: {
+    client: 'mysql2',
+    connection: {
+      host: 'localhost',
+      port: 3306,
+      user: 'root',
+      password: 'password',
+      database: 'sakila',
+      multipleStatements: true,
+    },
+    pool: { min: 0, max: 5 },
+  },
+  sqlite: {
+    client: 'sqlite3',
+    connection: {
+      filename: './mydb.sqlite3',
+    },
+    useNullAsDefault: true,
+    pool: { min: 0, max: 5 },
+  },
+};
+
+function getKnexConfig({ dbName, dbType }: { dbName: string; dbType: string }) {
+  const config = knexConfig[dbType];
+
+  if (dbType === 'sqlite') {
+    config.connection.filename = `./${dbName}.sqlite3`;
+    return config;
+  }
+  config.connection.database = dbName;
+  return config;
+}
+
+export { getKnexConfig };

tests/playwright/tests/utils/general.ts

@@ -50,4 +50,14 @@ function getDefaultPwd() {
   return 'Password123.';
 }
 
-export { getTextExcludeIconText, isSubset, getIconText, getDefaultPwd };
+function getBrowserTimezoneOffset() {
+  // get timezone offset
+  const timezoneOffset = new Date().getTimezoneOffset();
+  const hours = Math.floor(Math.abs(timezoneOffset) / 60);
+  const minutes = Math.abs(timezoneOffset % 60);
+  const sign = timezoneOffset <= 0 ? '+' : '-';
+  const formattedOffset = `${sign}${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}`;
+  return formattedOffset;
+}
+
+export { getTextExcludeIconText, isSubset, getIconText, getDefaultPwd, getBrowserTimezoneOffset };