Browse Source

Merge pull request #9265 from nocodb/nc-feat/enable-sql-integration-in-oss

Nc feat/enable sql integration in oss
pull/9303/head
Ramesh Mane 4 months ago committed by GitHub
parent
commit
08a87aba4d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 52
      packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue
  2. 10
      packages/nc-gui/components/dashboard/settings/data-sources/EditBase.vue
  3. 27
      packages/nc-gui/components/general/IntegrationIcon.vue
  4. 6
      packages/nc-gui/components/smartsheet/details/Api.vue
  5. 64
      packages/nc-gui/components/workspace/integrations/ConnectionsTab.vue
  6. 2
      packages/nc-gui/components/workspace/integrations/EditOrAdd.vue
  7. 6
      packages/nc-gui/components/workspace/integrations/IntegrationsTab.vue
  8. 31
      packages/nc-gui/components/workspace/integrations/forms/EditOrAddDatabase.vue
  9. 13
      packages/nc-gui/composables/useIntegrationsStore.ts
  10. 2
      packages/nc-gui/composables/useViewAggregate.ts
  11. 4
      packages/nc-gui/lang/en.json
  12. 4
      packages/nc-gui/store/workspace.ts
  13. 82
      packages/nc-gui/utils/syncDataUtils.ts
  14. 8
      packages/nocodb/src/models/Integration.ts
  15. 34
      packages/nocodb/src/services/integrations.service.ts

52
packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue

@ -1,6 +1,6 @@
<script lang="ts" setup>
import { Form, message } from 'ant-design-vue'
import { validateAndExtractSSLProp } from 'nocodb-sdk'
import { type IntegrationType, validateAndExtractSSLProp } from 'nocodb-sdk'
import {
ClientType,
type DatabricksConnection,
@ -233,8 +233,8 @@ const createSource = async () => {
emit('sourceCreated')
vOpen.value = false
creatingSource.value = false
} else if (status === JobStatus.FAILED) {
message.error('Failed to create base')
} else if (data.status === JobStatus.FAILED) {
message.error(data?.data?.error?.message || 'Failed to create base')
creatingSource.value = false
}
}
@ -358,8 +358,8 @@ const allowDataWrite = computed({
const changeIntegration = (triggerTestConnection = false) => {
if (formState.value.fk_integration_id && selectedIntegration.value) {
formState.value.dataSource = {
client: selectedIntegration.value.sub_type,
connection: {
client: selectedIntegration.value.sub_type,
database: selectedIntegrationDb.value,
},
searchPath: selectedIntegration.value.config?.searchPath,
@ -419,6 +419,22 @@ function handleAutoScroll(scroll: boolean, className: string) {
}
const filterIntegrationCategory = (c: IntegrationCategoryItemType) => [IntegrationCategoryType.DATABASE].includes(c.value)
const isIntgrationDisabled = (integration: IntegrationType = {}) => {
switch (integration.sub_type) {
case ClientType.SQLITE:
return {
isDisabled: integration?.source_count && integration.source_count > 0,
msg: 'Sqlite support only 1 database per connection',
}
default:
return {
isDisabled: false,
msg: '',
}
}
}
</script>
<template>
@ -516,16 +532,32 @@ const filterIntegrationCategory = (c: IntegrationCategoryItemType) => [Integrati
dropdown-match-select-width
@change="changeIntegration()"
>
<a-select-option v-for="integration in integrations" :key="integration.id" :value="integration.id">
<a-select-option
v-for="integration in integrations"
:key="integration.id"
:value="integration.id"
:disabled="isIntgrationDisabled(integration).isDisabled"
>
<div class="w-full flex gap-2 items-center" :data-testid="integration.title">
<GeneralBaseLogo
<GeneralIntegrationIcon
v-if="integration?.sub_type"
:source-type="integration.sub_type"
class="flex-none h-4 w-4"
:type="integration.sub_type"
:style="{
filter: isIntgrationDisabled(integration).isDisabled
? 'grayscale(100%) brightness(115%)'
: undefined,
}"
/>
<NcTooltip class="flex-1 truncate" show-on-truncate-only>
<NcTooltip
class="flex-1 truncate"
:show-on-truncate-only="!isIntgrationDisabled(integration).isDisabled"
>
<template #title>
{{ integration.title }}
{{
isIntgrationDisabled(integration).isDisabled
? isIntgrationDisabled(integration).msg
: integration.title
}}
</template>
{{ integration.title }}
</NcTooltip>

10
packages/nc-gui/components/dashboard/settings/data-sources/EditBase.vue

@ -413,10 +413,12 @@ function handleAutoScroll(scroll: boolean, className: string) {
>
<a-select-option v-for="integration in integrations" :key="integration.id" :value="integration.id">
<div class="w-full flex gap-2 items-center" :data-testid="integration.title">
<GeneralBaseLogo
v-if="integration.type"
:source-type="integration.sub_type"
class="flex-none h-4 w-4"
<GeneralIntegrationIcon
v-if="integration?.sub_type"
:type="integration.sub_type"
:style="{
filter: 'grayscale(100%) brightness(115%)',
}"
/>
<NcTooltip class="flex-1 truncate" show-on-truncate-only>
<template #title>

27
packages/nc-gui/components/general/IntegrationIcon.vue

@ -0,0 +1,27 @@
<script lang="ts" setup>
const props = withDefaults(
defineProps<{
type: keyof typeof allIntegrationsMapByValue
size: 'sx' | 'sm' | 'md' | 'lg' | 'xl' | 'xxl'
}>(),
{
size: 'sm',
},
)
</script>
<template>
<component
:is="allIntegrationsMapByValue[props.type]?.icon"
v-if="allIntegrationsMapByValue[props.type]?.icon"
class="stroke-transparent flex-none"
:class="{
'w-3.5 h-3.5': size === 'sx',
'w-4 h-4': size === 'sm',
'w-5 h-5': size === 'md',
'w-6 h-6': size === 'lg',
'w-7 h-7': size === 'xl',
'w-8 h-8': size === 'xxl',
}"
/>
</template>

6
packages/nc-gui/components/smartsheet/details/Api.vue

@ -176,12 +176,6 @@ const supportedDocs = [
title: string
href: string
}[]
const handleNavigateToDocs = (href: string) => {
navigateTo(href, {
open: navigateToBlankTargetOpenOption,
})
}
</script>
<template>

64
packages/nc-gui/components/workspace/integrations/ConnectionsTab.vue

@ -22,6 +22,8 @@ const { $api, $e } = useNuxtApp()
const { allCollaborators } = storeToRefs(useWorkspace())
const { bases } = storeToRefs(useBases())
const isDeleteIntegrationModalOpen = ref(false)
const toBeDeletedIntegration = ref<
| (IntegrationType & {
@ -139,7 +141,26 @@ const openDeleteIntegration = async (source: IntegrationType) => {
}
const onDeleteConfirm = async () => {
await deleteIntegration(toBeDeletedIntegration.value, true)
const isDeleted = await deleteIntegration(toBeDeletedIntegration.value, true)
if (isDeleted) {
for (const source of toBeDeletedIntegration.value?.sources || []) {
if (!source.base_id || !source.id || (source.base_id && !bases.value.get(source.base_id))) {
continue
}
const base = bases.value.get(source.base_id)
if (!Array.isArray(base?.sources)) {
continue
}
bases.value.set(source.base_id, {
...(base || {}),
sources: [...base.sources.filter((s) => s.id !== source.id)],
})
}
}
}
const loadOrgUsers = async () => {
@ -400,12 +421,7 @@ onKeyStroke('ArrowDown', onDown)
<td class="cell-type">
<div>
<NcBadge rounded="lg" class="flex items-center gap-2 px-0 py-1 !h-7 truncate !border-transparent">
<WorkspaceIntegrationsIcon
v-if="integration.sub_type"
:integration-type="integration.sub_type"
size="xs"
class="!p-0 !bg-transparent"
/>
<GeneralIntegrationIcon :type="integration.sub_type" />
<NcTooltip placement="bottom" show-on-truncate-only class="text-sm truncate">
<template #title> {{ clientTypesMap[integration?.sub_type]?.text || integration?.sub_type }}</template>
@ -506,10 +522,30 @@ onKeyStroke('ArrowDown', onDown)
<GeneralIcon class="text-gray-800" icon="edit" />
<span>{{ $t('general.edit') }}</span>
</NcMenuItem>
<NcMenuItem @click="duplicateIntegration(integration)">
<GeneralIcon class="text-gray-800" icon="duplicate" />
<span>{{ $t('general.duplicate') }}</span>
</NcMenuItem>
<NcTooltip :disabled="integration?.sub_type !== ClientType.SQLITE">
<template #title>
Not allowed for type
{{
integration.sub_type && clientTypesMap[integration.sub_type]
? clientTypesMap[integration.sub_type]?.text
: integration.sub_type
}}
</template>
<NcMenuItem
@click="duplicateIntegration(integration)"
:disabled="integration?.sub_type === ClientType.SQLITE"
>
<GeneralIcon
:class="{
'text-current': integration?.sub_type === ClientType.SQLITE,
'text-gray-800': integration?.sub_type !== ClientType.SQLITE,
}"
icon="duplicate"
/>
<span>{{ $t('general.duplicate') }}</span>
</NcMenuItem>
</NcTooltip>
<NcDivider />
<NcMenuItem class="!text-red-500 !hover:bg-red-50" @click="openDeleteIntegration(integration)">
<GeneralIcon icon="delete" />
@ -604,11 +640,7 @@ onKeyStroke('ArrowDown', onDown)
</template>
<div v-else-if="toBeDeletedIntegration" class="w-full flex flex-col text-gray-800">
<div class="flex flex-row items-center py-2 px-3.25 bg-gray-50 rounded-lg text-gray-700 mb-4">
<WorkspaceIntegrationsIcon
:integration-type="toBeDeletedIntegration.sub_type"
size="xs"
class="!p-0 !bg-transparent"
/>
<GeneralIntegrationIcon :type="toBeDeletedIntegration.sub_type" />
<div
class="text-ellipsis overflow-hidden select-none w-full pl-3"
:style="{ wordBreak: 'keep-all', whiteSpace: 'nowrap', display: 'inline' }"

2
packages/nc-gui/components/workspace/integrations/EditOrAdd.vue

@ -26,6 +26,8 @@ const connectionType = computed(() => {
return ClientType.PG
case integrationType.MySQL:
return ClientType.MYSQL
case integrationType.SQLITE:
return ClientType.SQLITE
default: {
return undefined
}

6
packages/nc-gui/components/workspace/integrations/IntegrationsTab.vue

@ -59,8 +59,12 @@ const upvotesData = computed(() => {
const getIntegrationsByCategory = (category: IntegrationCategoryType, query: string) => {
return allIntegrations.filter((i) => {
const isOssOnly = isEeUI ? !i?.isOssOnly : true
return (
filterIntegration(i) && i.categories.includes(category) && t(i.title).toLowerCase().includes(query.trim().toLowerCase())
isOssOnly &&
filterIntegration(i) &&
i.categories.includes(category) &&
t(i.title).toLowerCase().includes(query.trim().toLowerCase())
)
})
}

31
packages/nc-gui/components/workspace/integrations/forms/EditOrAddDatabase.vue

@ -54,6 +54,10 @@ const _getDefaultConnectionConfig = (client = ClientType.MYSQL) => {
if ('database' in config.connection) {
config.connection.database = ''
}
if (client === ClientType.SQLITE && config.connection?.connection?.filename) {
config.connection.connection.filename = ''
}
return config
}
@ -428,6 +432,14 @@ function handleAutoScroll(scroll: boolean, className: string) {
}
}
const activeIntegrationIcon = computed(() => {
const activeIntegrationType = isEditMode.value
? activeIntegration.value?.sub_type || activeIntegration.value?.config?.client
: activeIntegration.value?.type
return allIntegrationsMapByValue[activeIntegrationType]?.icon
})
// reset test status on config change
watch(
formState,
@ -504,12 +516,14 @@ watch(
>
<GeneralIcon icon="arrowLeft" />
</NcButton>
<WorkspaceIntegrationsIcon
:integration-type="
isEditMode ? activeIntegration?.sub_type || activeIntegration?.config?.client : activeIntegration?.type
"
size="xs"
/>
<div
v-if="activeIntegrationIcon"
class="h-8 w-8 flex items-center justify-center children:flex-none bg-gray-200 rounded-lg"
>
<component :is="activeIntegrationIcon" class="!stroke-transparent w-4 h-4" />
</div>
<div class="flex-1 text-base font-weight-700">{{ activeIntegration?.title }}</div>
</div>
<div class="flex items-center gap-3">
@ -656,7 +670,10 @@ watch(
:label="$t('labels.sqliteFile')"
v-bind="validateInfos['dataSource.connection.connection.filename']"
>
<a-input v-model:value="(formState.dataSource.connection as SQLiteConnection).connection.filename" />
<a-input
v-model:value="(formState.dataSource.connection as SQLiteConnection).connection.filename"
placeholder="Enter absolute file path"
/>
</a-form-item>
</a-col>
</a-row>

13
packages/nc-gui/composables/useIntegrationsStore.ts

@ -10,9 +10,10 @@ enum IntegrationsPageMode {
EDIT,
}
const integrationType: Record<'PostgreSQL' | 'MySQL', ClientType> = {
const integrationType: Record<'PostgreSQL' | 'MySQL' | 'SQLITE', ClientType> = {
PostgreSQL: ClientType.PG,
MySQL: ClientType.MYSQL,
SQLITE: ClientType.SQLITE,
}
type IntegrationsSubType = (typeof integrationType)[keyof typeof integrationType]
@ -43,6 +44,16 @@ function defaultValues(type: IntegrationsSubType) {
'class': 'logo',
}),
}
case integrationType.SQLITE:
return {
...genericValues,
type: integrationType.SQLITE,
title: 'SQLite',
logo: h(GeneralBaseLogo, {
'source-type': 'sqlite3',
'class': 'logo',
}),
}
}
}

2
packages/nc-gui/composables/useViewAggregate.ts

@ -25,8 +25,6 @@ const [useProvideViewAggregate, useViewAggregate] = useInjectionState(
const { nestedFilters } = useSmartsheetStoreOrThrow()
const { isUIAllowed } = useRoles()
const { fetchAggregatedData } = useSharedView()
const aggregations = ref({}) as Ref<Record<string, any>>

4
packages/nc-gui/lang/en.json

@ -377,7 +377,7 @@
"zendesk": "Zendesk",
"mysql": "MySQL",
"postgreSQL": "PostgreSQL",
"sqlServer": "SQL Server",
"sqlite": "SQLite",
"dataBricks": "DataBricks",
"mssqlServer": "MSSQL Server",
"oracle": "Oracle",
@ -825,7 +825,7 @@
"lengthValue": "Length/ value",
"dbType": "Database Type",
"servername": "servername / hostAddr",
"sqliteFile": "SQLite File",
"sqliteFile": "SQLite file path",
"hostAddress": "Host address",
"port": "Port number",
"username": "Username",

4
packages/nc-gui/store/workspace.ts

@ -20,6 +20,8 @@ export const useWorkspace = defineStore('workspaceStore', () => {
const collaborators = ref<any[] | null>()
const allCollaborators = ref<any[] | null>()
const router = useRouter()
const route = router.currentRoute
@ -296,6 +298,7 @@ export const useWorkspace = defineStore('workspaceStore', () => {
removeCollaborator,
updateCollaborator,
collaborators,
allCollaborators,
isInvitingCollaborators,
isCollaboratorsLoading,
addToFavourite,
@ -323,7 +326,6 @@ export const useWorkspace = defineStore('workspaceStore', () => {
auditLogsQuery,
audits,
auditPaginationData,
loadAudits,
isIntegrationsPageOpened,
navigateToIntegrations,

82
packages/nc-gui/utils/syncDataUtils.ts

@ -8,15 +8,13 @@ export interface IntegrationItemType {
categories: IntegrationCategoryType[]
isAvailable?: boolean
iconStyle?: CSSProperties
isOssOnly?: boolean
}
export interface IntegrationCategoryItemType {
title: string
subtitle: string
value: IntegrationCategoryType
icon: FunctionalComponent<SVGAttributes, {}, any, {}>
iconBgColor?: string
iconStyle?: CSSProperties
isAvailable?: boolean
teleEventName?: IntegrationCategoryType
}
@ -26,133 +24,68 @@ export const integrationCategories: IntegrationCategoryItemType[] = [
title: 'labels.database',
subtitle: 'objects.integrationCategories.databaseSubtitle',
value: IntegrationCategoryType.DATABASE,
icon: iconMap.database,
iconBgColor: '#D4F7E0',
iconStyle: {
color: '#17803D',
},
isAvailable: true,
},
{
title: 'objects.integrationCategories.ai',
subtitle: 'objects.integrationCategories.ai',
value: IntegrationCategoryType.AI,
icon: iconMap.openai,
iconBgColor: '#FFF0F7',
iconStyle: {
color: '#801044',
},
},
{
title: 'objects.integrationCategories.communication',
subtitle: 'objects.integrationCategories.communicationSubtitle',
value: IntegrationCategoryType.COMMUNICATION,
icon: iconMap.messageCircle,
iconBgColor: '#FFF0F7',
iconStyle: {
color: '#801044',
},
},
{
title: 'objects.integrationCategories.spreadSheet',
subtitle: 'objects.integrationCategories.spreadSheetSubtitle',
value: IntegrationCategoryType.SPREAD_SHEET,
teleEventName: IntegrationCategoryType.OTHERS,
icon: iconMap.viewGannt,
iconBgColor: '#FFF0D1',
iconStyle: {
color: '#977223',
},
},
{
title: 'objects.integrationCategories.projectManagement',
subtitle: 'objects.integrationCategories.projectManagementSubtitle',
value: IntegrationCategoryType.PROJECT_MANAGEMENT,
icon: iconMap.viewGannt,
iconBgColor: '#FFF0D1',
iconStyle: {
color: '#977223',
},
},
{
title: 'objects.integrationCategories.ticketing',
subtitle: 'objects.integrationCategories.ticketingSubtitle',
value: IntegrationCategoryType.TICKETING,
icon: iconMap.globe,
iconBgColor: '#FFF0D1',
iconStyle: {
color: '#977223',
},
},
{
title: 'objects.integrationCategories.crm',
subtitle: 'objects.integrationCategories.crmSubtitle',
value: IntegrationCategoryType.CRM,
icon: iconMap.users,
iconBgColor: '#D7F2FF',
iconStyle: {
color: '#207399',
},
},
{
title: 'objects.integrationCategories.marketing',
subtitle: 'objects.integrationCategories.marketingSubtitle',
value: IntegrationCategoryType.MARKETING,
icon: iconMap.heart,
iconBgColor: '#FED8F4',
iconStyle: {
color: '#972377',
},
},
{
title: 'objects.integrationCategories.ats',
subtitle: 'objects.integrationCategories.atsSubtitle',
value: IntegrationCategoryType.ATS,
icon: iconMap.multiFile,
iconBgColor: '#FEE6D6',
iconStyle: {
color: '#C86827',
},
},
{
title: 'objects.integrationCategories.development',
subtitle: 'objects.integrationCategories.developmentSubtitle',
value: IntegrationCategoryType.DEVELOPMENT,
icon: iconMap.code,
iconBgColor: '#E5D4F5',
iconStyle: {
color: '#4B177B',
},
},
{
title: 'objects.integrationCategories.finance',
subtitle: 'objects.integrationCategories.financeSubtitle',
value: IntegrationCategoryType.FINANCE,
icon: iconMap.dollerSign,
iconBgColor: '#D4F7E0',
iconStyle: {
color: '#17803D',
},
},
{
title: 'labels.storage',
subtitle: 'objects.integrationCategories.storageSubtitle',
value: IntegrationCategoryType.STORAGE,
icon: iconMap.ncSave,
iconBgColor: '#E7E7E9',
iconStyle: {
color: '#374151',
},
},
{
title: 'objects.integrationCategories.others',
subtitle: 'objects.integrationCategories.othersSubtitle',
value: IntegrationCategoryType.OTHERS,
icon: iconMap.plusSquare,
iconBgColor: 'white',
iconStyle: {
color: '#374151',
},
},
]
@ -176,6 +109,14 @@ export const allIntegrations: IntegrationItemType[] = [
categories: [IntegrationCategoryType.DATABASE],
isAvailable: true,
},
{
title: 'objects.syncData.sqlite',
value: ClientType.SQLITE,
icon: iconMap.sqlServer,
categories: [IntegrationCategoryType.DATABASE],
isAvailable: true,
isOssOnly: true,
},
{
title: 'objects.syncData.snowflake',
value: ClientType.SNOWFLAKE,
@ -494,3 +435,8 @@ export const allIntegrations: IntegrationItemType[] = [
// categories: [IntegrationCategoryType.OTHERS],
// },
]
export const allIntegrationsMapByValue = allIntegrations.reduce((acc, curr) => {
acc[curr.value] = curr
return acc
}, {} as Record<string, IntegrationItemType>)

8
packages/nocodb/src/models/Integration.ts

@ -171,6 +171,7 @@ export default class Integration implements IntegrationType {
userId: string;
includeDatabaseInfo?: boolean;
type?: IntegrationsType;
sub_type?: string | ClientTypes;
limit?: number;
offset?: number;
includeSourceCount?: boolean;
@ -199,6 +200,10 @@ export default class Integration implements IntegrationType {
if (args.type) {
qb.where(`${MetaTable.INTEGRATIONS}.type`, args.type);
}
// if sub_type is provided then filter integrations based on sub_type
if (args.sub_type) {
qb.where(`${MetaTable.INTEGRATIONS}.sub_type`, args.sub_type);
}
qb.where((whereQb) => {
whereQb
@ -247,6 +252,9 @@ export default class Integration implements IntegrationType {
integration.config = partialExtract(config, [
'client',
['connection', 'database'],
// extract params related to sqlite
['connection', 'filepath'],
['connection', 'connection', 'filepath'],
['searchPath'],
]);
}

34
packages/nocodb/src/services/integrations.service.ts

@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { AppEvents } from 'nocodb-sdk';
import type { IntegrationReqType, IntegrationsType } from 'nocodb-sdk';
import { AppEvents, ClientType } from 'nocodb-sdk';
import { IntegrationsType } from 'nocodb-sdk';
import type { IntegrationReqType } from 'nocodb-sdk';
import type { NcContext, NcRequest } from '~/interface/config';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { validatePayload } from '~/helpers';
@ -227,6 +228,35 @@ export class IntegrationsService {
}
param.logger?.('Creating the integration');
// for SQLite check for existing integration which refers to the same file
if (integrationBody.sub_type === 'sqlite3') {
// get all integrations of type sqlite3
const integrations = await Integration.list({
userId: param.req.user?.id,
includeDatabaseInfo: true,
type: IntegrationsType.Database,
sub_type: ClientType.SQLITE,
limit: 1000,
offset: 0,
includeSourceCount: false,
query: '',
});
if (integrations.list && integrations.list.length > 0) {
for (const integration of integrations.list) {
const config = integration.config as any;
if (
(config?.connection?.filename ||
config?.connection?.connection?.filename) ===
(integrationBody.config?.connection?.filename ||
integrationBody.config?.connection?.connection?.filename)
) {
NcError.badRequest('Integration with same file already exists');
}
}
}
}
const integration = await Integration.createIntegration({
...integrationBody,
...(param.integration.copy_from_id

Loading…
Cancel
Save