From 477a96fc910de658d833835155c19e15c2cb919c Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 18 Apr 2024 21:12:46 +0000 Subject: [PATCH] feat: better source create ux --- packages/nc-gui/components/general/Modal.vue | 4 +- .../lib/databricks/DatabricksClient.ts | 75 +++++-------------- .../src/db/sql-migrator/lib/KnexMigratorv2.ts | 8 ++ packages/nocodb/src/helpers/columnHelpers.ts | 14 +++- packages/nocodb/src/helpers/syncMigration.ts | 3 +- packages/nocodb/src/models/Source.ts | 10 +-- .../jobs/at-import/at-import.processor.ts | 7 +- .../jobs/jobs/export-import/import.service.ts | 3 +- .../source-create/source-create.processor.ts | 29 ++++--- .../nocodb/src/services/columns.service.ts | 10 ++- .../nocodb/src/services/meta-diffs.service.ts | 12 +-- .../nocodb/src/services/sources.service.ts | 39 ++++++---- .../nocodb/src/services/tables.service.ts | 12 +-- .../nocodb/src/utils/nc-config/constants.ts | 1 + 14 files changed, 114 insertions(+), 113 deletions(-) diff --git a/packages/nc-gui/components/general/Modal.vue b/packages/nc-gui/components/general/Modal.vue index 0f50715729..069559f70f 100644 --- a/packages/nc-gui/components/general/Modal.vue +++ b/packages/nc-gui/components/general/Modal.vue @@ -20,8 +20,8 @@ const props = withDefaults( const emits = defineEmits(['update:visible']) -const { width: propWidth, destroyOnClose } = props -const { maskClosable, closable, keyboard } = toRefs(props) +const { width: propWidth } = props +const { maskClosable, closable, keyboard, destroyOnClose } = toRefs(props) const width = computed(() => { if (propWidth) { diff --git a/packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts b/packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts index b28e2c83ff..1d415e5f4f 100644 --- a/packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts +++ b/packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts @@ -372,70 +372,30 @@ class DatabricksClient extends KnexClient { const _func = this.createDatabaseIfNotExists.name; const result = new Result(); log.api(`${_func}:args:`, args); - let tempSqlClient; - - return; try { - const connectionParamsWithoutDb = JSON.parse( - JSON.stringify(this.connectionConfig), - ); - let rows = []; - try { - connectionParamsWithoutDb.connection.database = 'postgres'; - tempSqlClient = knex({ - ...connectionParamsWithoutDb, - pool: { min: 0, max: 1 }, - }); + // check if catalog exists + const catalogs = await this.sqlClient.raw(`SHOW CATALOGS LIKE ?`, [ + args.database, + ]); - log.debug('checking if db exists'); - rows = ( - await tempSqlClient.raw( - `SELECT datname as database FROM pg_database WHERE datistemplate = false and datname = ?`, - [args.database], - ) - ).rows; - } catch (e) { - log.debug('checking if db exists'); - rows = ( - await this.sqlClient.raw( - `SELECT datname as database FROM pg_database WHERE datistemplate = false and datname = ?`, - [args.database], - ) - ).rows; - } - if (rows.length === 0) { - log.debug('creating database:', args); - await tempSqlClient.raw(`CREATE DATABASE ?? ENCODING 'UTF8'`, [ - args.database, - ]); + if (catalogs.length === 0) { + throw new Error( + 'We do not support creating catalogs yet, please use an existing catalog', + ); } - const schemaName = this.connectionConfig.schema || 'default'; - - // Check schemaExists because `CREATE SCHEMA IF NOT EXISTS` requires permissions of `CREATE ON DATABASE` - const schemaExists = !!( - await this.sqlClient.raw( - `SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?`, - [schemaName], - ) - ).rows?.[0]; + // check if database exists + const databases = await this.sqlClient.raw(`SHOW DATABASES LIKE ?`, [ + args.schema, + ]); - if (!schemaExists) { - await this.sqlClient.raw( - `CREATE SCHEMA IF NOT EXISTS ?? AUTHORIZATION ?? `, - [schemaName, this.connectionConfig.connection.user], - ); + if (databases.length === 0) { + await this.sqlClient.raw(`CREATE DATABASE ${args.schema}`); } - - // this.sqlClient = knex(this.connectionConfig); } catch (e) { log.ppe(e, _func); throw e; - } finally { - if (tempSqlClient) { - await tempSqlClient.destroy(); - } } log.api(`${_func}: result`, result); @@ -2555,7 +2515,12 @@ class DatabricksClient extends KnexClient { } get schema(): string { - return (this.connectionConfig && this.connectionConfig.schema) || 'default'; + return ( + (this.connectionConfig && + this.connectionConfig.connection && + this.connectionConfig.connection.schema) || + 'default' + ); } /** diff --git a/packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts b/packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts index c603be0f62..f9f7d99a36 100644 --- a/packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts +++ b/packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts @@ -397,6 +397,14 @@ export default class KnexMigratorv2 { database: connectionConfig.connection.database, schema: source.getConfig()?.schema, }); + } else if (source.type === 'databricks') { + this.emit( + `${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}`, + ); + await sqlClient.createDatabaseIfNotExists({ + database: connectionConfig.connection.database, + schema: connectionConfig.connection.schema, + }); } else if (connectionConfig.client !== 'sqlite3') { this.emit( `${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}`, diff --git a/packages/nocodb/src/helpers/columnHelpers.ts b/packages/nocodb/src/helpers/columnHelpers.ts index ade72bbf60..b5c697cd6a 100644 --- a/packages/nocodb/src/helpers/columnHelpers.ts +++ b/packages/nocodb/src/helpers/columnHelpers.ts @@ -21,6 +21,7 @@ import { GridViewColumn } from '~/models'; import validateParams from '~/helpers/validateParams'; import { getUniqueColumnAliasName } from '~/helpers/getUniqueName'; import Column from '~/models/Column'; +import { DriverClient } from '~/utils/nc-config'; export const randomID = customAlphabet( '1234567890abcdefghijklmnopqrstuvwxyz_', @@ -370,12 +371,19 @@ export async function populateRollupForLTAR({ await GridViewColumn.update(viewCol.id, { show: false }); } -export const sanitizeColumnName = (name: string) => { +export const sanitizeColumnName = (name: string, sourceType?: DriverClient) => { if (process.env.NC_SANITIZE_COLUMN_NAME === 'false') return name; - const columnName = name.replace(/\W/g, '_'); + let columnName = name.replace(/\W/g, '_'); // if column name only contains _ then return as 'field' - if (/^_+$/.test(columnName)) return 'field'; + if (/^_+$/.test(columnName)) columnName = 'field'; + + if (sourceType) { + if (sourceType === DriverClient.DATABRICKS) { + // databricks column name should be lowercase + columnName = columnName.toLowerCase(); + } + } return columnName; }; diff --git a/packages/nocodb/src/helpers/syncMigration.ts b/packages/nocodb/src/helpers/syncMigration.ts index e0868f21f2..3d4331b3af 100644 --- a/packages/nocodb/src/helpers/syncMigration.ts +++ b/packages/nocodb/src/helpers/syncMigration.ts @@ -35,7 +35,6 @@ export async function syncBaseMigration( await migrator.migrationsUp({ source }); } catch (e) { - console.log(e); - // throw e; + throw e; } } diff --git a/packages/nocodb/src/models/Source.ts b/packages/nocodb/src/models/Source.ts index d63e5c968c..af47691751 100644 --- a/packages/nocodb/src/models/Source.ts +++ b/packages/nocodb/src/models/Source.ts @@ -1,11 +1,9 @@ import { UITypes } from 'nocodb-sdk'; import CryptoJS from 'crypto-js'; import { v4 as uuidv4 } from 'uuid'; +import type { DriverClient } from '~/utils/nc-config'; import type { BoolType, SourceType } from 'nocodb-sdk'; -import type { DB_TYPES } from '~/utils/globals'; -import Model from '~/models/Model'; -import Base from '~/models/Base'; -import SyncSource from '~/models/SyncSource'; +import { Base, Model, SyncSource } from '~/models'; import NocoCache from '~/cache/NocoCache'; import { CacheDelDirection, @@ -29,7 +27,7 @@ export default class Source implements SourceType { id?: string; base_id?: string; alias?: string; - type?: (typeof DB_TYPES)[number]; + type?: DriverClient; is_meta?: BoolType; config?: string; inflection_column?: string; @@ -39,7 +37,7 @@ export default class Source implements SourceType { enabled?: BoolType; meta?: any; - constructor(source: Partial) { + constructor(source: Partial) { Object.assign(this, source); } diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 353d5eca17..b3e9f5c1d3 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -320,7 +320,7 @@ export class AtImportProcessor { const uniqueFieldNameGen = getUniqueNameGenerator('field', table_name); // truncate to 50 chars if character if exceeds above 50 - const col_name = sanitizeColumnName(name)?.slice(0, 50); + const col_name = sanitizeColumnName(name, getRootDbType())?.slice(0, 50); // for knex, replace . with _ const col_alias = name.trim().replace(/\./g, '_'); @@ -498,7 +498,10 @@ export class AtImportProcessor { // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; table.title = tblSchema[i].name; - let sanitizedName = sanitizeColumnName(tblSchema[i].name); + let sanitizedName = sanitizeColumnName( + tblSchema[i].name, + getRootDbType(), + ); // truncate to 50 chars if character if exceeds above 50 // upto 64 should be fine but we are keeping it to 50 since diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts index 04cc370256..3789e52705 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts @@ -187,7 +187,8 @@ export class ImportService { const colRef = modelData.columns.find( (a) => a.column_name && - sanitizeColumnName(a.column_name) === col.column_name, + sanitizeColumnName(a.column_name, source.type) === + col.column_name, ); idMap.set(colRef.id, col.id); diff --git a/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts b/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts index cd5b9bfdac..09ed262b94 100644 --- a/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts @@ -25,19 +25,28 @@ export class SourceCreateProcessor { this.debugLog(log); }; - const createdBase = await this.sourcesService.baseCreate({ - baseId, - source, - logger: logBasic, - req, - }); - - if (createdBase.isMeta()) { - delete createdBase.config; + const { source: createdSource, error } = + await this.sourcesService.baseCreate({ + baseId, + source, + logger: logBasic, + req, + }); + + if (error) { + await this.sourcesService.baseDelete({ + sourceId: createdSource.id, + req: {}, + }); + throw error; + } + + if (createdSource.isMeta()) { + delete createdSource.config; } this.debugLog(`job completed for ${job.id}`); - return createdBase; + return createdSource; } } diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index b896d90904..dbd49a8aa5 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -165,7 +165,10 @@ export class ColumnsService { const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); if (!isVirtualCol(param.column)) { - param.column.column_name = sanitizeColumnName(param.column.column_name); + param.column.column_name = sanitizeColumnName( + param.column.column_name, + source.type, + ); } // trim leading and trailing spaces from column title as knex trim them by default @@ -1458,7 +1461,10 @@ export class ColumnsService { const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); if (!isVirtualCol(param.column)) { - param.column.column_name = sanitizeColumnName(param.column.column_name); + param.column.column_name = sanitizeColumnName( + param.column.column_name, + source.type, + ); } // trim leading and trailing spaces from column title as knex trim them by default diff --git a/packages/nocodb/src/services/meta-diffs.service.ts b/packages/nocodb/src/services/meta-diffs.service.ts index df4ff7bda6..fa3c572b16 100644 --- a/packages/nocodb/src/services/meta-diffs.service.ts +++ b/packages/nocodb/src/services/meta-diffs.service.ts @@ -141,8 +141,6 @@ export class MetaDiffsService { const changes: Array = []; const virtualRelationColumns: Column[] = []; - const isDatabricks = sqlClient.knex.clientType() === 'databricks'; - // @ts-ignore const tableList: Array<{ tn: string }> = ( await sqlClient.tableList({ schema: source.getConfig()?.schema }) @@ -180,10 +178,7 @@ export class MetaDiffsService { if (table.tn === 'nc_evolutions') continue; const oldMetaIdx = oldTableMetas.findIndex( - (m) => - m.table_name === table.tn || - (isDatabricks && - m.table_name.toLowerCase() === table.tn.toLowerCase()), + (m) => m.table_name === table.tn, ); // new table @@ -228,10 +223,7 @@ export class MetaDiffsService { for (const column of colListRef[table.tn]) { const oldColIdx = oldMeta.columns.findIndex( - (c) => - c.column_name === column.cn || - (isDatabricks && - c.column_name.toLowerCase() === column.cn.toLowerCase()), + (c) => c.column_name === column.cn, ); // new table diff --git a/packages/nocodb/src/services/sources.service.ts b/packages/nocodb/src/services/sources.service.ts index 9c96d25c3b..bc9ec7c4b0 100644 --- a/packages/nocodb/src/services/sources.service.ts +++ b/packages/nocodb/src/services/sources.service.ts @@ -83,13 +83,18 @@ export class SourcesService { source: BaseReqType; logger?: (message: string) => void; req: NcRequest; - }) { + }): Promise<{ + source: Source; + error?: any; + }> { validatePayload('swagger.json#/components/schemas/BaseReq', param.source); // type | base | baseId const baseBody = param.source; const base = await Base.getWithInfo(param.baseId); + let error; + param.logger?.('Creating the source'); const source = await Source.createBase({ @@ -98,26 +103,30 @@ export class SourcesService { baseId: base.id, }); - await syncBaseMigration(base, source); + try { + await syncBaseMigration(base, source); - param.logger?.('Populating meta'); + param.logger?.('Populating meta'); - const info = await populateMeta(source, base, param.logger); + const info = await populateMeta(source, base, param.logger); - await populateRollupColumnAndHideLTAR(source, base); + await populateRollupColumnAndHideLTAR(source, base); - this.appHooksService.emit(AppEvents.APIS_CREATED, { - info, - req: param.req, - }); + this.appHooksService.emit(AppEvents.APIS_CREATED, { + info, + req: param.req, + }); - delete source.config; + delete source.config; - this.appHooksService.emit(AppEvents.BASE_CREATE, { - source, - req: param.req, - }); + this.appHooksService.emit(AppEvents.BASE_CREATE, { + source, + req: param.req, + }); + } catch (e) { + error = e; + } - return source; + return { source, error }; } } diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index e7645ebe22..abb45e87e9 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -77,7 +77,9 @@ export class TablesService { } if (source.type === 'databricks') { - param.table.table_name = param.table.table_name.replace(/\s/g, '_'); + param.table.table_name = param.table.table_name + .replace(/\s/g, '_') + .toLowerCase(); } if (source.isMeta(true) && base.prefix && !source.isMeta(true, 1)) { @@ -517,10 +519,9 @@ export class TablesService { } if (source.type === 'databricks') { - tableCreatePayLoad.table_name = tableCreatePayLoad.table_name.replace( - /\s/g, - '_', - ); + tableCreatePayLoad.table_name = tableCreatePayLoad.table_name + .replace(/\s/g, '_') + .toLowerCase(); } if (source.is_meta && base.prefix) { @@ -605,6 +606,7 @@ export class TablesService { // - 5 is a buffer for suffix column.column_name = sanitizeColumnName( column.column_name.slice(0, mxColumnLength - 5), + source.type, ); if (uniqueColumnNameCount[column.column_name]) { diff --git a/packages/nocodb/src/utils/nc-config/constants.ts b/packages/nocodb/src/utils/nc-config/constants.ts index 98bc4f4d1c..5a2fe8adb4 100644 --- a/packages/nocodb/src/utils/nc-config/constants.ts +++ b/packages/nocodb/src/utils/nc-config/constants.ts @@ -77,6 +77,7 @@ export const knownQueryParams = [ export enum DriverClient { MYSQL = 'mysql2', + MYSQL_LEGACY = 'mysql', MSSQL = 'mssql', PG = 'pg', SQLITE = 'sqlite3',