Merge pull request #9265 from nocodb/nc-feat/enable-sql-integration-in-oss

Nc feat/enable sql integration in oss

packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue

@@ -1,6 +1,6 @@
 <script lang="ts" setup>
 import { Form, message } from 'ant-design-vue'
-import { validateAndExtractSSLProp } from 'nocodb-sdk'
+import { type IntegrationType, validateAndExtractSSLProp } from 'nocodb-sdk'
 import {
   ClientType,
   type DatabricksConnection,
@@ -233,8 +233,8 @@ const createSource = async () => {
             emit('sourceCreated')
             vOpen.value = false
             creatingSource.value = false
-          } else if (status === JobStatus.FAILED) {
+          } else if (data.status === JobStatus.FAILED) {
-            message.error('Failed to create base')
+            message.error(data?.data?.error?.message || 'Failed to create base')
             creatingSource.value = false
           }
         }
@@ -358,8 +358,8 @@ const allowDataWrite = computed({
 const changeIntegration = (triggerTestConnection = false) => {
   if (formState.value.fk_integration_id && selectedIntegration.value) {
     formState.value.dataSource = {
-      connection: {
       client: selectedIntegration.value.sub_type,
+      connection: {
         database: selectedIntegrationDb.value,
       },
       searchPath: selectedIntegration.value.config?.searchPath,
@@ -419,6 +419,22 @@ function handleAutoScroll(scroll: boolean, className: string) {
 }
 
 const filterIntegrationCategory = (c: IntegrationCategoryItemType) => [IntegrationCategoryType.DATABASE].includes(c.value)
+
+const isIntgrationDisabled = (integration: IntegrationType = {}) => {
+  switch (integration.sub_type) {
+    case ClientType.SQLITE:
+      return {
+        isDisabled: integration?.source_count && integration.source_count > 0,
+        msg: 'Sqlite support only 1 database per connection',
+      }
+
+    default:
+      return {
+        isDisabled: false,
+        msg: '',
+      }
+  }
+}
 </script>
 
 <template>
@@ -516,16 +532,32 @@ const filterIntegrationCategory = (c: IntegrationCategoryItemType) => [Integrati
                           dropdown-match-select-width
                           @change="changeIntegration()"
                         >
-                          <a-select-option v-for="integration in integrations" :key="integration.id" :value="integration.id">
+                          <a-select-option
+                            v-for="integration in integrations"
+                            :key="integration.id"
+                            :value="integration.id"
+                            :disabled="isIntgrationDisabled(integration).isDisabled"
+                          >
                             <div class="w-full flex gap-2 items-center" :data-testid="integration.title">
-                              <GeneralBaseLogo
+                              <GeneralIntegrationIcon
                                 v-if="integration?.sub_type"
-                                :source-type="integration.sub_type"
+                                :type="integration.sub_type"
-                                class="flex-none h-4 w-4"
+                                :style="{
+                                  filter: isIntgrationDisabled(integration).isDisabled
+                                    ? 'grayscale(100%) brightness(115%)'
+                                    : undefined,
+                                }"
                               />
-                              <NcTooltip class="flex-1 truncate" show-on-truncate-only>
+                              <NcTooltip
+                                class="flex-1 truncate"
+                                :show-on-truncate-only="!isIntgrationDisabled(integration).isDisabled"
+                              >
                                 <template #title>
-                                  {{ integration.title }}
+                                  {{
+                                    isIntgrationDisabled(integration).isDisabled
+                                      ? isIntgrationDisabled(integration).msg
+                                      : integration.title
+                                  }}
                                 </template>
                                 {{ integration.title }}
                               </NcTooltip>

packages/nc-gui/components/dashboard/settings/data-sources/EditBase.vue

@@ -413,10 +413,12 @@ function handleAutoScroll(scroll: boolean, className: string) {
                       >
                         <a-select-option v-for="integration in integrations" :key="integration.id" :value="integration.id">
                           <div class="w-full flex gap-2 items-center" :data-testid="integration.title">
-                            <GeneralBaseLogo
+                            <GeneralIntegrationIcon
-                              v-if="integration.type"
+                              v-if="integration?.sub_type"
-                              :source-type="integration.sub_type"
+                              :type="integration.sub_type"
-                              class="flex-none h-4 w-4"
+                              :style="{
+                                filter: 'grayscale(100%) brightness(115%)',
+                              }"
                             />
                             <NcTooltip class="flex-1 truncate" show-on-truncate-only>
                               <template #title>

packages/nc-gui/components/general/IntegrationIcon.vue

@@ -0,0 +1,27 @@
+<script lang="ts" setup>
+const props = withDefaults(
+  defineProps<{
+    type: keyof typeof allIntegrationsMapByValue
+    size: 'sx' | 'sm' | 'md' | 'lg' | 'xl' | 'xxl'
+  }>(),
+  {
+    size: 'sm',
+  },
+)
+</script>
+
+<template>
+  <component
+    :is="allIntegrationsMapByValue[props.type]?.icon"
+    v-if="allIntegrationsMapByValue[props.type]?.icon"
+    class="stroke-transparent flex-none"
+    :class="{
+      'w-3.5 h-3.5': size === 'sx',
+      'w-4 h-4': size === 'sm',
+      'w-5 h-5': size === 'md',
+      'w-6 h-6': size === 'lg',
+      'w-7 h-7': size === 'xl',
+      'w-8 h-8': size === 'xxl',
+    }"
+  />
+</template>

packages/nc-gui/components/smartsheet/details/Api.vue

@@ -176,12 +176,6 @@ const supportedDocs = [
   title: string
   href: string
 }[]
-
-const handleNavigateToDocs = (href: string) => {
-  navigateTo(href, {
-    open: navigateToBlankTargetOpenOption,
-  })
-}
 </script>
 
 <template>

packages/nc-gui/components/workspace/integrations/ConnectionsTab.vue

@@ -22,6 +22,8 @@ const { $api, $e } = useNuxtApp()
 
 const { allCollaborators } = storeToRefs(useWorkspace())
 
+const { bases } = storeToRefs(useBases())
+
 const isDeleteIntegrationModalOpen = ref(false)
 const toBeDeletedIntegration = ref<
   | (IntegrationType & {
@@ -139,7 +141,26 @@ const openDeleteIntegration = async (source: IntegrationType) => {
 }
 
 const onDeleteConfirm = async () => {
-  await deleteIntegration(toBeDeletedIntegration.value, true)
+  const isDeleted = await deleteIntegration(toBeDeletedIntegration.value, true)
+
+  if (isDeleted) {
+    for (const source of toBeDeletedIntegration.value?.sources || []) {
+      if (!source.base_id || !source.id || (source.base_id && !bases.value.get(source.base_id))) {
+        continue
+      }
+
+      const base = bases.value.get(source.base_id)
+
+      if (!Array.isArray(base?.sources)) {
+        continue
+      }
+
+      bases.value.set(source.base_id, {
+        ...(base || {}),
+        sources: [...base.sources.filter((s) => s.id !== source.id)],
+      })
+    }
+  }
 }
 
 const loadOrgUsers = async () => {
@@ -400,12 +421,7 @@ onKeyStroke('ArrowDown', onDown)
                 <td class="cell-type">
                   <div>
                     <NcBadge rounded="lg" class="flex items-center gap-2 px-0 py-1 !h-7 truncate !border-transparent">
-                      <WorkspaceIntegrationsIcon
+                      <GeneralIntegrationIcon :type="integration.sub_type" />
-                        v-if="integration.sub_type"
-                        :integration-type="integration.sub_type"
-                        size="xs"
-                        class="!p-0 !bg-transparent"
-                      />
                       <NcTooltip placement="bottom" show-on-truncate-only class="text-sm truncate">
                         <template #title> {{ clientTypesMap[integration?.sub_type]?.text || integration?.sub_type }}</template>
 
@@ -506,10 +522,30 @@ onKeyStroke('ArrowDown', onDown)
                             <GeneralIcon class="text-gray-800" icon="edit" />
                             <span>{{ $t('general.edit') }}</span>
                           </NcMenuItem>
-                          <NcMenuItem @click="duplicateIntegration(integration)">
+                          <NcTooltip :disabled="integration?.sub_type !== ClientType.SQLITE">
-                            <GeneralIcon class="text-gray-800" icon="duplicate" />
+                            <template #title>
+                              Not allowed for type
+                              {{
+                                integration.sub_type && clientTypesMap[integration.sub_type]
+                                  ? clientTypesMap[integration.sub_type]?.text
+                                  : integration.sub_type
+                              }}
+                            </template>
+
+                            <NcMenuItem
+                              @click="duplicateIntegration(integration)"
+                              :disabled="integration?.sub_type === ClientType.SQLITE"
+                            >
+                              <GeneralIcon
+                                :class="{
+                                  'text-current': integration?.sub_type === ClientType.SQLITE,
+                                  'text-gray-800': integration?.sub_type !== ClientType.SQLITE,
+                                }"
+                                icon="duplicate"
+                              />
                               <span>{{ $t('general.duplicate') }}</span>
                             </NcMenuItem>
+                          </NcTooltip>
                           <NcDivider />
                           <NcMenuItem class="!text-red-500 !hover:bg-red-50" @click="openDeleteIntegration(integration)">
                             <GeneralIcon icon="delete" />
@@ -604,11 +640,7 @@ onKeyStroke('ArrowDown', onDown)
         </template>
         <div v-else-if="toBeDeletedIntegration" class="w-full flex flex-col text-gray-800">
           <div class="flex flex-row items-center py-2 px-3.25 bg-gray-50 rounded-lg text-gray-700 mb-4">
-            <WorkspaceIntegrationsIcon
+            <GeneralIntegrationIcon :type="toBeDeletedIntegration.sub_type" />
-              :integration-type="toBeDeletedIntegration.sub_type"
-              size="xs"
-              class="!p-0 !bg-transparent"
-            />
             <div
               class="text-ellipsis overflow-hidden select-none w-full pl-3"
               :style="{ wordBreak: 'keep-all', whiteSpace: 'nowrap', display: 'inline' }"

packages/nc-gui/components/workspace/integrations/EditOrAdd.vue

@@ -26,6 +26,8 @@ const connectionType = computed(() => {
       return ClientType.PG
     case integrationType.MySQL:
       return ClientType.MYSQL
+    case integrationType.SQLITE:
+      return ClientType.SQLITE
     default: {
       return undefined
     }

packages/nc-gui/components/workspace/integrations/IntegrationsTab.vue

@@ -59,8 +59,12 @@ const upvotesData = computed(() => {
 
 const getIntegrationsByCategory = (category: IntegrationCategoryType, query: string) => {
   return allIntegrations.filter((i) => {
+    const isOssOnly = isEeUI ? !i?.isOssOnly : true
     return (
-      filterIntegration(i) && i.categories.includes(category) && t(i.title).toLowerCase().includes(query.trim().toLowerCase())
+      isOssOnly &&
+      filterIntegration(i) &&
+      i.categories.includes(category) &&
+      t(i.title).toLowerCase().includes(query.trim().toLowerCase())
     )
   })
 }

packages/nc-gui/components/workspace/integrations/forms/EditOrAddDatabase.vue

@@ -54,6 +54,10 @@ const _getDefaultConnectionConfig = (client = ClientType.MYSQL) => {
   if ('database' in config.connection) {
     config.connection.database = ''
   }
+
+  if (client === ClientType.SQLITE && config.connection?.connection?.filename) {
+    config.connection.connection.filename = ''
+  }
   return config
 }
 
@@ -428,6 +432,14 @@ function handleAutoScroll(scroll: boolean, className: string) {
   }
 }
 
+const activeIntegrationIcon = computed(() => {
+  const activeIntegrationType = isEditMode.value
+    ? activeIntegration.value?.sub_type || activeIntegration.value?.config?.client
+    : activeIntegration.value?.type
+
+  return allIntegrationsMapByValue[activeIntegrationType]?.icon
+})
+
 // reset test status on config change
 watch(
   formState,
@@ -504,12 +516,14 @@ watch(
         >
           <GeneralIcon icon="arrowLeft" />
         </NcButton>
-        <WorkspaceIntegrationsIcon
+
-          :integration-type="
+        <div
-            isEditMode ? activeIntegration?.sub_type || activeIntegration?.config?.client : activeIntegration?.type
+          v-if="activeIntegrationIcon"
-          "
+          class="h-8 w-8 flex items-center justify-center children:flex-none bg-gray-200 rounded-lg"
-          size="xs"
+        >
-        />
+          <component :is="activeIntegrationIcon" class="!stroke-transparent w-4 h-4" />
+        </div>
+
         <div class="flex-1 text-base font-weight-700">{{ activeIntegration?.title }}</div>
       </div>
       <div class="flex items-center gap-3">
@@ -656,7 +670,10 @@ watch(
                           :label="$t('labels.sqliteFile')"
                           v-bind="validateInfos['dataSource.connection.connection.filename']"
                         >
-                          <a-input v-model:value="(formState.dataSource.connection as SQLiteConnection).connection.filename" />
+                          <a-input
+                            v-model:value="(formState.dataSource.connection as SQLiteConnection).connection.filename"
+                            placeholder="Enter absolute file path"
+                          />
                         </a-form-item>
                       </a-col>
                     </a-row>

packages/nc-gui/composables/useIntegrationsStore.ts

@@ -10,9 +10,10 @@ enum IntegrationsPageMode {
   EDIT,
 }
 
-const integrationType: Record<'PostgreSQL' | 'MySQL', ClientType> = {
+const integrationType: Record<'PostgreSQL' | 'MySQL' | 'SQLITE', ClientType> = {
   PostgreSQL: ClientType.PG,
   MySQL: ClientType.MYSQL,
+  SQLITE: ClientType.SQLITE,
 }
 
 type IntegrationsSubType = (typeof integrationType)[keyof typeof integrationType]
@@ -43,6 +44,16 @@ function defaultValues(type: IntegrationsSubType) {
           'class': 'logo',
         }),
       }
+    case integrationType.SQLITE:
+      return {
+        ...genericValues,
+        type: integrationType.SQLITE,
+        title: 'SQLite',
+        logo: h(GeneralBaseLogo, {
+          'source-type': 'sqlite3',
+          'class': 'logo',
+        }),
+      }
   }
 }
 

packages/nc-gui/composables/useViewAggregate.ts

@@ -25,8 +25,6 @@ const [useProvideViewAggregate, useViewAggregate] = useInjectionState(
 
     const { nestedFilters } = useSmartsheetStoreOrThrow()
 
-    const { isUIAllowed } = useRoles()
-
     const { fetchAggregatedData } = useSharedView()
 
     const aggregations = ref({}) as Ref<Record<string, any>>

packages/nc-gui/lang/en.json

@@ -377,7 +377,7 @@
       "zendesk": "Zendesk",
       "mysql": "MySQL",
       "postgreSQL": "PostgreSQL",
-      "sqlServer": "SQL Server",
+      "sqlite": "SQLite",
       "dataBricks": "DataBricks",
       "mssqlServer": "MSSQL Server",
       "oracle": "Oracle",
@@ -825,7 +825,7 @@
     "lengthValue": "Length/ value",
     "dbType": "Database Type",
     "servername": "servername / hostAddr",
-    "sqliteFile": "SQLite File",
+    "sqliteFile": "SQLite file path",
     "hostAddress": "Host address",
     "port": "Port number",
     "username": "Username",

packages/nc-gui/store/workspace.ts

@@ -20,6 +20,8 @@ export const useWorkspace = defineStore('workspaceStore', () => {
 
   const collaborators = ref<any[] | null>()
 
+  const allCollaborators = ref<any[] | null>()
+
   const router = useRouter()
 
   const route = router.currentRoute
@@ -296,6 +298,7 @@ export const useWorkspace = defineStore('workspaceStore', () => {
     removeCollaborator,
     updateCollaborator,
     collaborators,
+    allCollaborators,
     isInvitingCollaborators,
     isCollaboratorsLoading,
     addToFavourite,
@@ -323,7 +326,6 @@ export const useWorkspace = defineStore('workspaceStore', () => {
     auditLogsQuery,
     audits,
     auditPaginationData,
-
     loadAudits,
     isIntegrationsPageOpened,
     navigateToIntegrations,

packages/nc-gui/utils/syncDataUtils.ts

@@ -8,15 +8,13 @@ export interface IntegrationItemType {
   categories: IntegrationCategoryType[]
   isAvailable?: boolean
   iconStyle?: CSSProperties
+  isOssOnly?: boolean
 }
 
 export interface IntegrationCategoryItemType {
   title: string
   subtitle: string
   value: IntegrationCategoryType
-  icon: FunctionalComponent<SVGAttributes, {}, any, {}>
-  iconBgColor?: string
-  iconStyle?: CSSProperties
   isAvailable?: boolean
   teleEventName?: IntegrationCategoryType
 }
@@ -26,133 +24,68 @@ export const integrationCategories: IntegrationCategoryItemType[] = [
     title: 'labels.database',
     subtitle: 'objects.integrationCategories.databaseSubtitle',
     value: IntegrationCategoryType.DATABASE,
-    icon: iconMap.database,
-    iconBgColor: '#D4F7E0',
-    iconStyle: {
-      color: '#17803D',
-    },
     isAvailable: true,
   },
   {
     title: 'objects.integrationCategories.ai',
     subtitle: 'objects.integrationCategories.ai',
     value: IntegrationCategoryType.AI,
-    icon: iconMap.openai,
-    iconBgColor: '#FFF0F7',
-    iconStyle: {
-      color: '#801044',
-    },
   },
   {
     title: 'objects.integrationCategories.communication',
     subtitle: 'objects.integrationCategories.communicationSubtitle',
     value: IntegrationCategoryType.COMMUNICATION,
-    icon: iconMap.messageCircle,
-    iconBgColor: '#FFF0F7',
-    iconStyle: {
-      color: '#801044',
-    },
   },
   {
     title: 'objects.integrationCategories.spreadSheet',
     subtitle: 'objects.integrationCategories.spreadSheetSubtitle',
     value: IntegrationCategoryType.SPREAD_SHEET,
     teleEventName: IntegrationCategoryType.OTHERS,
-    icon: iconMap.viewGannt,
-    iconBgColor: '#FFF0D1',
-    iconStyle: {
-      color: '#977223',
-    },
   },
   {
     title: 'objects.integrationCategories.projectManagement',
     subtitle: 'objects.integrationCategories.projectManagementSubtitle',
     value: IntegrationCategoryType.PROJECT_MANAGEMENT,
-    icon: iconMap.viewGannt,
-    iconBgColor: '#FFF0D1',
-    iconStyle: {
-      color: '#977223',
-    },
   },
   {
     title: 'objects.integrationCategories.ticketing',
     subtitle: 'objects.integrationCategories.ticketingSubtitle',
     value: IntegrationCategoryType.TICKETING,
-    icon: iconMap.globe,
-    iconBgColor: '#FFF0D1',
-    iconStyle: {
-      color: '#977223',
-    },
   },
   {
     title: 'objects.integrationCategories.crm',
     subtitle: 'objects.integrationCategories.crmSubtitle',
     value: IntegrationCategoryType.CRM,
-    icon: iconMap.users,
-    iconBgColor: '#D7F2FF',
-    iconStyle: {
-      color: '#207399',
-    },
   },
   {
     title: 'objects.integrationCategories.marketing',
     subtitle: 'objects.integrationCategories.marketingSubtitle',
     value: IntegrationCategoryType.MARKETING,
-    icon: iconMap.heart,
-    iconBgColor: '#FED8F4',
-    iconStyle: {
-      color: '#972377',
-    },
   },
   {
     title: 'objects.integrationCategories.ats',
     subtitle: 'objects.integrationCategories.atsSubtitle',
     value: IntegrationCategoryType.ATS,
-    icon: iconMap.multiFile,
-    iconBgColor: '#FEE6D6',
-    iconStyle: {
-      color: '#C86827',
-    },
   },
   {
     title: 'objects.integrationCategories.development',
     subtitle: 'objects.integrationCategories.developmentSubtitle',
     value: IntegrationCategoryType.DEVELOPMENT,
-    icon: iconMap.code,
-    iconBgColor: '#E5D4F5',
-    iconStyle: {
-      color: '#4B177B',
-    },
   },
   {
     title: 'objects.integrationCategories.finance',
     subtitle: 'objects.integrationCategories.financeSubtitle',
     value: IntegrationCategoryType.FINANCE,
-    icon: iconMap.dollerSign,
-    iconBgColor: '#D4F7E0',
-    iconStyle: {
-      color: '#17803D',
-    },
   },
   {
     title: 'labels.storage',
     subtitle: 'objects.integrationCategories.storageSubtitle',
     value: IntegrationCategoryType.STORAGE,
-    icon: iconMap.ncSave,
-    iconBgColor: '#E7E7E9',
-    iconStyle: {
-      color: '#374151',
-    },
   },
   {
     title: 'objects.integrationCategories.others',
     subtitle: 'objects.integrationCategories.othersSubtitle',
     value: IntegrationCategoryType.OTHERS,
-    icon: iconMap.plusSquare,
-    iconBgColor: 'white',
-    iconStyle: {
-      color: '#374151',
-    },
   },
 ]
 
@@ -176,6 +109,14 @@ export const allIntegrations: IntegrationItemType[] = [
     categories: [IntegrationCategoryType.DATABASE],
     isAvailable: true,
   },
+  {
+    title: 'objects.syncData.sqlite',
+    value: ClientType.SQLITE,
+    icon: iconMap.sqlServer,
+    categories: [IntegrationCategoryType.DATABASE],
+    isAvailable: true,
+    isOssOnly: true,
+  },
   {
     title: 'objects.syncData.snowflake',
     value: ClientType.SNOWFLAKE,
@@ -494,3 +435,8 @@ export const allIntegrations: IntegrationItemType[] = [
   //   categories: [IntegrationCategoryType.OTHERS],
   // },
 ]
+
+export const allIntegrationsMapByValue = allIntegrations.reduce((acc, curr) => {
+  acc[curr.value] = curr
+  return acc
+}, {} as Record<string, IntegrationItemType>)

packages/nocodb/src/models/Integration.ts

@@ -171,6 +171,7 @@ export default class Integration implements IntegrationType {
       userId: string;
       includeDatabaseInfo?: boolean;
       type?: IntegrationsType;
+      sub_type?: string | ClientTypes;
       limit?: number;
       offset?: number;
       includeSourceCount?: boolean;
@@ -199,6 +200,10 @@ export default class Integration implements IntegrationType {
     if (args.type) {
       qb.where(`${MetaTable.INTEGRATIONS}.type`, args.type);
     }
+    // if sub_type is provided then filter integrations based on sub_type
+    if (args.sub_type) {
+      qb.where(`${MetaTable.INTEGRATIONS}.sub_type`, args.sub_type);
+    }
 
     qb.where((whereQb) => {
       whereQb
@@ -247,6 +252,9 @@ export default class Integration implements IntegrationType {
         integration.config = partialExtract(config, [
           'client',
           ['connection', 'database'],
+          // extract params related to sqlite
+          ['connection', 'filepath'],
+          ['connection', 'connection', 'filepath'],
           ['searchPath'],
         ]);
       }

packages/nocodb/src/services/integrations.service.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common';
-import { AppEvents } from 'nocodb-sdk';
+import { AppEvents, ClientType } from 'nocodb-sdk';
-import type { IntegrationReqType, IntegrationsType } from 'nocodb-sdk';
+import { IntegrationsType } from 'nocodb-sdk';
+import type { IntegrationReqType } from 'nocodb-sdk';
 import type { NcContext, NcRequest } from '~/interface/config';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import { validatePayload } from '~/helpers';
@@ -227,6 +228,35 @@ export class IntegrationsService {
     }
     param.logger?.('Creating the integration');
 
+    // for SQLite check for existing integration which refers to the same file
+    if (integrationBody.sub_type === 'sqlite3') {
+      // get all integrations of type sqlite3
+      const integrations = await Integration.list({
+        userId: param.req.user?.id,
+        includeDatabaseInfo: true,
+        type: IntegrationsType.Database,
+        sub_type: ClientType.SQLITE,
+        limit: 1000,
+        offset: 0,
+        includeSourceCount: false,
+        query: '',
+      });
+
+      if (integrations.list && integrations.list.length > 0) {
+        for (const integration of integrations.list) {
+          const config = integration.config as any;
+          if (
+            (config?.connection?.filename ||
+              config?.connection?.connection?.filename) ===
+            (integrationBody.config?.connection?.filename ||
+              integrationBody.config?.connection?.connection?.filename)
+          ) {
+            NcError.badRequest('Integration with same file already exists');
+          }
+        }
+      }
+    }
+
     const integration = await Integration.createIntegration({
       ...integrationBody,
       ...(param.integration.copy_from_id