feat: better source create ux

packages/nc-gui/components/general/Modal.vue

@@ -20,8 +20,8 @@ const props = withDefaults(
 
 const emits = defineEmits(['update:visible'])
 
-const { width: propWidth, destroyOnClose } = props
+const { width: propWidth } = props
-const { maskClosable, closable, keyboard } = toRefs(props)
+const { maskClosable, closable, keyboard, destroyOnClose } = toRefs(props)
 
 const width = computed(() => {
   if (propWidth) {

packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts

@@ -372,70 +372,30 @@ class DatabricksClient extends KnexClient {
     const _func = this.createDatabaseIfNotExists.name;
     const result = new Result();
     log.api(`${_func}:args:`, args);
-    let tempSqlClient;
-
-    return;
 
     try {
-      const connectionParamsWithoutDb = JSON.parse(
+      // check if catalog exists
-        JSON.stringify(this.connectionConfig),
+      const catalogs = await this.sqlClient.raw(`SHOW CATALOGS LIKE ?`, [
-      );
+        args.database,
-      let rows = [];
+      ]);
-      try {
-        connectionParamsWithoutDb.connection.database = 'postgres';
-        tempSqlClient = knex({
-          ...connectionParamsWithoutDb,
-          pool: { min: 0, max: 1 },
-        });
 
-        log.debug('checking if db exists');
+      if (catalogs.length === 0) {
-        rows = (
+        throw new Error(
-          await tempSqlClient.raw(
+          'We do not support creating catalogs yet, please use an existing catalog',
-            `SELECT datname as database FROM pg_database WHERE datistemplate = false and datname = ?`,
+        );
-            [args.database],
-          )
-        ).rows;
-      } catch (e) {
-        log.debug('checking if db exists');
-        rows = (
-          await this.sqlClient.raw(
-            `SELECT datname as database FROM pg_database WHERE datistemplate = false and datname = ?`,
-            [args.database],
-          )
-        ).rows;
-      }
-      if (rows.length === 0) {
-        log.debug('creating database:', args);
-        await tempSqlClient.raw(`CREATE DATABASE ?? ENCODING 'UTF8'`, [
-          args.database,
-        ]);
       }
 
-      const schemaName = this.connectionConfig.schema || 'default';
+      // check if database exists
-
+      const databases = await this.sqlClient.raw(`SHOW DATABASES LIKE ?`, [
-      // Check schemaExists because `CREATE SCHEMA IF NOT EXISTS` requires permissions of `CREATE ON DATABASE`
+        args.schema,
-      const schemaExists = !!(
+      ]);
-        await this.sqlClient.raw(
-          `SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?`,
-          [schemaName],
-        )
-      ).rows?.[0];
 
-      if (!schemaExists) {
+      if (databases.length === 0) {
-        await this.sqlClient.raw(
+        await this.sqlClient.raw(`CREATE DATABASE ${args.schema}`);
-          `CREATE SCHEMA IF NOT EXISTS ??  AUTHORIZATION ?? `,
-          [schemaName, this.connectionConfig.connection.user],
-        );
       }
-
-      // this.sqlClient = knex(this.connectionConfig);
     } catch (e) {
       log.ppe(e, _func);
       throw e;
-    } finally {
-      if (tempSqlClient) {
-        await tempSqlClient.destroy();
-      }
     }
 
     log.api(`${_func}: result`, result);
@@ -2555,7 +2515,12 @@ class DatabricksClient extends KnexClient {
   }
 
   get schema(): string {
-    return (this.connectionConfig && this.connectionConfig.schema) || 'default';
+    return (
+      (this.connectionConfig &&
+        this.connectionConfig.connection &&
+        this.connectionConfig.connection.schema) ||
+      'default'
+    );
   }
 
   /**

packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts

@@ -397,6 +397,14 @@ export default class KnexMigratorv2 {
         database: connectionConfig.connection.database,
         schema: source.getConfig()?.schema,
       });
+    } else if (source.type === 'databricks') {
+      this.emit(
+        `${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}`,
+      );
+      await sqlClient.createDatabaseIfNotExists({
+        database: connectionConfig.connection.database,
+        schema: connectionConfig.connection.schema,
+      });
     } else if (connectionConfig.client !== 'sqlite3') {
       this.emit(
         `${connectionConfig.client}: Creating DB if not exists ${connectionConfig.connection.database}`,

packages/nocodb/src/helpers/columnHelpers.ts

@@ -21,6 +21,7 @@ import { GridViewColumn } from '~/models';
 import validateParams from '~/helpers/validateParams';
 import { getUniqueColumnAliasName } from '~/helpers/getUniqueName';
 import Column from '~/models/Column';
+import { DriverClient } from '~/utils/nc-config';
 
 export const randomID = customAlphabet(
   '1234567890abcdefghijklmnopqrstuvwxyz_',
@@ -370,12 +371,19 @@ export async function populateRollupForLTAR({
   await GridViewColumn.update(viewCol.id, { show: false });
 }
 
-export const sanitizeColumnName = (name: string) => {
+export const sanitizeColumnName = (name: string, sourceType?: DriverClient) => {
   if (process.env.NC_SANITIZE_COLUMN_NAME === 'false') return name;
-  const columnName = name.replace(/\W/g, '_');
+  let columnName = name.replace(/\W/g, '_');
 
   // if column name only contains _ then return as 'field'
-  if (/^_+$/.test(columnName)) return 'field';
+  if (/^_+$/.test(columnName)) columnName = 'field';
+
+  if (sourceType) {
+    if (sourceType === DriverClient.DATABRICKS) {
+      // databricks column name should be lowercase
+      columnName = columnName.toLowerCase();
+    }
+  }
 
   return columnName;
 };

packages/nocodb/src/helpers/syncMigration.ts

@@ -35,7 +35,6 @@ export async function syncBaseMigration(
 
     await migrator.migrationsUp({ source });
   } catch (e) {
-    console.log(e);
+    throw e;
-    // throw e;
   }
 }

packages/nocodb/src/models/Source.ts

@@ -1,11 +1,9 @@
 import { UITypes } from 'nocodb-sdk';
 import CryptoJS from 'crypto-js';
 import { v4 as uuidv4 } from 'uuid';
+import type { DriverClient } from '~/utils/nc-config';
 import type { BoolType, SourceType } from 'nocodb-sdk';
-import type { DB_TYPES } from '~/utils/globals';
+import { Base, Model, SyncSource } from '~/models';
-import Model from '~/models/Model';
-import Base from '~/models/Base';
-import SyncSource from '~/models/SyncSource';
 import NocoCache from '~/cache/NocoCache';
 import {
   CacheDelDirection,
@@ -29,7 +27,7 @@ export default class Source implements SourceType {
   id?: string;
   base_id?: string;
   alias?: string;
-  type?: (typeof DB_TYPES)[number];
+  type?: DriverClient;
   is_meta?: BoolType;
   config?: string;
   inflection_column?: string;
@@ -39,7 +37,7 @@ export default class Source implements SourceType {
   enabled?: BoolType;
   meta?: any;
 
-  constructor(source: Partial<Source>) {
+  constructor(source: Partial<SourceType>) {
     Object.assign(this, source);
   }
 

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -320,7 +320,7 @@ export class AtImportProcessor {
       const uniqueFieldNameGen = getUniqueNameGenerator('field', table_name);
 
       // truncate to 50 chars if character if exceeds above 50
-      const col_name = sanitizeColumnName(name)?.slice(0, 50);
+      const col_name = sanitizeColumnName(name, getRootDbType())?.slice(0, 50);
 
       // for knex, replace . with _
       const col_alias = name.trim().replace(/\./g, '_');
@@ -498,7 +498,10 @@ export class AtImportProcessor {
 
         // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id;
         table.title = tblSchema[i].name;
-        let sanitizedName = sanitizeColumnName(tblSchema[i].name);
+        let sanitizedName = sanitizeColumnName(
+          tblSchema[i].name,
+          getRootDbType(),
+        );
 
         // truncate to 50 chars if character if exceeds above 50
         // upto 64 should be fine but we are keeping it to 50 since

packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@@ -187,7 +187,8 @@ export class ImportService {
           const colRef = modelData.columns.find(
             (a) =>
               a.column_name &&
-              sanitizeColumnName(a.column_name) === col.column_name,
+              sanitizeColumnName(a.column_name, source.type) ===
+                col.column_name,
           );
           idMap.set(colRef.id, col.id);
 

packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts

@@ -25,19 +25,28 @@ export class SourceCreateProcessor {
       this.debugLog(log);
     };
 
-    const createdBase = await this.sourcesService.baseCreate({
+    const { source: createdSource, error } =
-      baseId,
+      await this.sourcesService.baseCreate({
-      source,
+        baseId,
-      logger: logBasic,
+        source,
-      req,
+        logger: logBasic,
-    });
+        req,
-
+      });
-    if (createdBase.isMeta()) {
+
-      delete createdBase.config;
+    if (error) {
+      await this.sourcesService.baseDelete({
+        sourceId: createdSource.id,
+        req: {},
+      });
+      throw error;
+    }
+
+    if (createdSource.isMeta()) {
+      delete createdSource.config;
     }
 
     this.debugLog(`job completed for ${job.id}`);
 
-    return createdBase;
+    return createdSource;
   }
 }

packages/nocodb/src/services/columns.service.ts

@@ -165,7 +165,10 @@ export class ColumnsService {
     const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType);
 
     if (!isVirtualCol(param.column)) {
-      param.column.column_name = sanitizeColumnName(param.column.column_name);
+      param.column.column_name = sanitizeColumnName(
+        param.column.column_name,
+        source.type,
+      );
     }
 
     // trim leading and trailing spaces from column title as knex trim them by default
@@ -1458,7 +1461,10 @@ export class ColumnsService {
       const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType);
 
       if (!isVirtualCol(param.column)) {
-        param.column.column_name = sanitizeColumnName(param.column.column_name);
+        param.column.column_name = sanitizeColumnName(
+          param.column.column_name,
+          source.type,
+        );
       }
 
       // trim leading and trailing spaces from column title as knex trim them by default

packages/nocodb/src/services/meta-diffs.service.ts

@@ -141,8 +141,6 @@ export class MetaDiffsService {
     const changes: Array<MetaDiff> = [];
     const virtualRelationColumns: Column<LinkToAnotherRecordColumn>[] = [];
 
-    const isDatabricks = sqlClient.knex.clientType() === 'databricks';
-
     // @ts-ignore
     const tableList: Array<{ tn: string }> = (
       await sqlClient.tableList({ schema: source.getConfig()?.schema })
@@ -180,10 +178,7 @@ export class MetaDiffsService {
       if (table.tn === 'nc_evolutions') continue;
 
       const oldMetaIdx = oldTableMetas.findIndex(
-        (m) =>
+        (m) => m.table_name === table.tn,
-          m.table_name === table.tn ||
-          (isDatabricks &&
-            m.table_name.toLowerCase() === table.tn.toLowerCase()),
       );
 
       // new table
@@ -228,10 +223,7 @@ export class MetaDiffsService {
 
       for (const column of colListRef[table.tn]) {
         const oldColIdx = oldMeta.columns.findIndex(
-          (c) =>
+          (c) => c.column_name === column.cn,
-            c.column_name === column.cn ||
-            (isDatabricks &&
-              c.column_name.toLowerCase() === column.cn.toLowerCase()),
         );
 
         // new table

packages/nocodb/src/services/sources.service.ts

@@ -83,13 +83,18 @@ export class SourcesService {
     source: BaseReqType;
     logger?: (message: string) => void;
     req: NcRequest;
-  }) {
+  }): Promise<{
+    source: Source;
+    error?: any;
+  }> {
     validatePayload('swagger.json#/components/schemas/BaseReq', param.source);
 
     // type | base | baseId
     const baseBody = param.source;
     const base = await Base.getWithInfo(param.baseId);
 
+    let error;
+
     param.logger?.('Creating the source');
 
     const source = await Source.createBase({
@@ -98,26 +103,30 @@ export class SourcesService {
       baseId: base.id,
     });
 
-    await syncBaseMigration(base, source);
+    try {
+      await syncBaseMigration(base, source);
 
-    param.logger?.('Populating meta');
+      param.logger?.('Populating meta');
 
-    const info = await populateMeta(source, base, param.logger);
+      const info = await populateMeta(source, base, param.logger);
 
-    await populateRollupColumnAndHideLTAR(source, base);
+      await populateRollupColumnAndHideLTAR(source, base);
 
-    this.appHooksService.emit(AppEvents.APIS_CREATED, {
+      this.appHooksService.emit(AppEvents.APIS_CREATED, {
-      info,
+        info,
-      req: param.req,
+        req: param.req,
-    });
+      });
 
-    delete source.config;
+      delete source.config;
 
-    this.appHooksService.emit(AppEvents.BASE_CREATE, {
+      this.appHooksService.emit(AppEvents.BASE_CREATE, {
-      source,
+        source,
-      req: param.req,
+        req: param.req,
-    });
+      });
+    } catch (e) {
+      error = e;
+    }
 
-    return source;
+    return { source, error };
   }
 }

packages/nocodb/src/services/tables.service.ts

@@ -77,7 +77,9 @@ export class TablesService {
     }
 
     if (source.type === 'databricks') {
-      param.table.table_name = param.table.table_name.replace(/\s/g, '_');
+      param.table.table_name = param.table.table_name
+        .replace(/\s/g, '_')
+        .toLowerCase();
     }
 
     if (source.isMeta(true) && base.prefix && !source.isMeta(true, 1)) {
@@ -517,10 +519,9 @@ export class TablesService {
     }
 
     if (source.type === 'databricks') {
-      tableCreatePayLoad.table_name = tableCreatePayLoad.table_name.replace(
+      tableCreatePayLoad.table_name = tableCreatePayLoad.table_name
-        /\s/g,
+        .replace(/\s/g, '_')
-        '_',
+        .toLowerCase();
-      );
     }
 
     if (source.is_meta && base.prefix) {
@@ -605,6 +606,7 @@ export class TablesService {
         // - 5 is a buffer for suffix
         column.column_name = sanitizeColumnName(
           column.column_name.slice(0, mxColumnLength - 5),
+          source.type,
         );
 
         if (uniqueColumnNameCount[column.column_name]) {

packages/nocodb/src/utils/nc-config/constants.ts

@@ -77,6 +77,7 @@ export const knownQueryParams = [
 
 export enum DriverClient {
   MYSQL = 'mysql2',
+  MYSQL_LEGACY = 'mysql',
   MSSQL = 'mssql',
   PG = 'pg',
   SQLITE = 'sqlite3',