Browse Source

feat: dbricks

nc-feat/dbricks-v3
mertmit 7 months ago
parent
commit
0a10c78374
  1. 34
      packages/nc-gui/components/dashboard/settings/data-sources/EditBase.vue
  2. 2
      packages/nc-gui/components/template/Editor.vue
  3. 1
      packages/nc-gui/lib/enums.ts
  4. 27
      packages/nc-gui/utils/baseCreateUtils.ts
  5. 820
      packages/nocodb-sdk/src/lib/sqlUi/DatabricksUi.ts
  6. 5
      packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts
  7. 18
      packages/nocodb/src/db/BaseModelSqlv2.ts
  8. 26
      packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
  9. 44
      packages/nocodb/src/db/functionMappings/databricks.ts
  10. 4
      packages/nocodb/src/db/mapFunctionName.ts
  11. 6
      packages/nocodb/src/db/sql-client/lib/SqlClientFactory.ts
  12. 2621
      packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts
  13. 3
      packages/nocodb/src/db/sql-client/lib/databricks/databricks.queries.ts
  14. 509
      packages/nocodb/src/db/sql-mgr/code/models/xc/ModelXcMetaDatabricks.ts
  15. 3
      packages/nocodb/src/db/sql-mgr/code/models/xc/ModelXcMetaFactory.ts
  16. 8
      packages/nocodb/src/helpers/populateMeta.ts
  17. 9
      packages/nocodb/src/schema/swagger-v2.json
  18. 12
      packages/nocodb/src/schema/swagger.json
  19. 12
      packages/nocodb/src/services/meta-diffs.service.ts
  20. 4
      packages/nocodb/src/services/tables.service.ts
  21. 1
      packages/nocodb/src/utils/globals.ts
  22. 1
      packages/nocodb/src/utils/nc-config/constants.ts

34
packages/nc-gui/components/dashboard/settings/data-sources/EditBase.vue

@ -437,6 +437,40 @@ onMounted(async () => {
</a-form-item>
</template>
<template v-else-if="formState.dataSource.client === ClientType.DATABRICKS">
<a-form-item label="Token" v-bind="validateInfos['dataSource.connection.token']">
<a-input
v-model:value="(formState.dataSource.connection as DatabricksConnection).token"
class="nc-extdb-host-token"
/>
</a-form-item>
<a-form-item label="Host" v-bind="validateInfos['dataSource.connection.host']">
<a-input
v-model:value="(formState.dataSource.connection as DatabricksConnection).host"
class="nc-extdb-host-address"
/>
</a-form-item>
<a-form-item label="Path" v-bind="validateInfos['dataSource.connection.path']">
<a-input v-model:value="(formState.dataSource.connection as DatabricksConnection).path" class="nc-extdb-host-path" />
</a-form-item>
<a-form-item label="Database" v-bind="validateInfos['dataSource.connection.database']">
<a-input
v-model:value="(formState.dataSource.connection as DatabricksConnection).database"
class="nc-extdb-host-database"
/>
</a-form-item>
<a-form-item label="Schema" v-bind="validateInfos['dataSource.connection.schema']">
<a-input
v-model:value="(formState.dataSource.connection as DatabricksConnection).schema"
class="nc-extdb-host-schema"
/>
</a-form-item>
</template>
<template v-else>
<!-- Host Address -->
<a-form-item :label="$t('labels.hostAddress')" v-bind="validateInfos['dataSource.connection.host']">

2
packages/nc-gui/components/template/Editor.vue

@ -306,7 +306,7 @@ function remapColNames(batchData: any[], columns: ColumnType[]) {
function missingRequiredColumnsValidation(tn: string) {
const missingRequiredColumns = columns.value.filter(
(c: Record<string, any>) =>
(c.pk ? !c.ai && !c.cdf : !c.cdf && c.rqd) &&
(c.pk ? !c.ai && !c.cdf && !c.meta?.ag : !c.cdf && c.rqd) &&
!srcDestMapping.value[tn].some((r: Record<string, any>) => r.destCn === c.title),
)

1
packages/nc-gui/lib/enums.ts

@ -5,6 +5,7 @@ export enum ClientType {
SQLITE = 'sqlite3',
VITESS = 'vitess',
SNOWFLAKE = 'snowflake',
DATABRICKS = 'databricks',
}
export enum Language {

27
packages/nc-gui/utils/baseCreateUtils.ts

@ -9,7 +9,7 @@ interface ProjectCreateForm {
title: string
dataSource: {
client: ClientType
connection: DefaultConnection | SQLiteConnection | SnowflakeConnection
connection: DefaultConnection | SQLiteConnection | SnowflakeConnection | DatabricksConnection
searchPath?: string[]
}
inflection: {
@ -47,6 +47,14 @@ interface SnowflakeConnection {
schema: string
}
interface DatabricksConnection {
token: string
host: string
path: string
database: string
schema: string
}
const defaultHost = 'localhost'
const testDataBaseNames = {
@ -84,12 +92,16 @@ export const clientTypes = [
text: 'Snowflake',
value: ClientType.SNOWFLAKE,
},
{
text: 'Databricks',
value: ClientType.DATABRICKS,
},
]
const homeDir = ''
type ConnectionClientType =
| Exclude<ClientType, ClientType.SQLITE | ClientType.SNOWFLAKE>
| Exclude<ClientType, ClientType.SQLITE | ClientType.SNOWFLAKE | ClientType.DATABRICKS>
| 'tidb'
| 'yugabyte'
| 'citusdb'
@ -99,7 +111,7 @@ type ConnectionClientType =
const sampleConnectionData: { [key in ConnectionClientType]: DefaultConnection } & { [ClientType.SQLITE]: SQLiteConnection } & {
[ClientType.SNOWFLAKE]: SnowflakeConnection
} = {
} & { [ClientType.DATABRICKS]: DatabricksConnection } = {
[ClientType.PG]: {
host: defaultHost,
port: '5432',
@ -144,6 +156,13 @@ const sampleConnectionData: { [key in ConnectionClientType]: DefaultConnection }
database: 'DATABASE',
schema: 'PUBLIC',
},
[ClientType.DATABRICKS]: {
token: 'dapiPLACEHOLDER',
host: 'PLACEHOLDER.cloud.databricks.com',
path: '/sql/1.0/warehouses/PLACEHOLDER',
database: 'database',
schema: 'default',
},
tidb: {
host: defaultHost,
port: '4000',
@ -215,4 +234,4 @@ enum CertTypes {
key = 'key',
}
export { SSLUsage, CertTypes, ProjectCreateForm, DefaultConnection, SQLiteConnection, SnowflakeConnection }
export { SSLUsage, CertTypes, ProjectCreateForm, DefaultConnection, SQLiteConnection, SnowflakeConnection, DatabricksConnection }

820
packages/nocodb-sdk/src/lib/sqlUi/DatabricksUi.ts

@ -0,0 +1,820 @@
import UITypes from '../UITypes';
import { IDType } from './index';
const dbTypes = [
'BIGINT',
'BINARY',
'BOOLEAN',
'DATE',
'DECIMAL',
'DOUBLE',
'FLOAT',
'INT',
'INTERVAL',
'VOID',
'SMALLINT',
'STRING',
'TIMESTAMP',
'TIMESTAMP_NTZ',
'TINYINT',
];
export class DatabricksUi {
static getNewTableColumns() {
return [
{
column_name: 'id',
title: 'Id',
dt: 'int',
dtx: 'int',
ct: 'int',
nrqd: false,
rqd: true,
ck: false,
pk: true,
un: false,
ai: true,
cdf: null,
clen: null,
np: null,
ns: 0,
dtxp: '',
dtxs: '',
altered: 1,
uidt: 'ID',
uip: '',
uicn: '',
},
{
column_name: 'title',
title: 'Title',
dt: 'string',
dtx: 'specificType',
ct: 'string',
nrqd: true,
rqd: false,
ck: false,
pk: false,
un: false,
ai: false,
cdf: null,
clen: 45,
np: null,
ns: null,
dtxp: '',
dtxs: '',
altered: 1,
uidt: 'SingleLineText',
uip: '',
uicn: '',
},
{
column_name: 'created_at',
title: 'CreatedAt',
dt: 'TIMESTAMP',
dtx: 'specificType',
ct: 'TIMESTAMP',
nrqd: true,
rqd: false,
ck: false,
pk: false,
un: false,
ai: false,
clen: 45,
np: null,
ns: null,
dtxp: '',
dtxs: '',
altered: 1,
uidt: UITypes.CreatedTime,
uip: '',
uicn: '',
system: true,
},
{
column_name: 'updated_at',
title: 'UpdatedAt',
dt: 'TIMESTAMP',
dtx: 'specificType',
ct: 'TIMESTAMP',
nrqd: true,
rqd: false,
ck: false,
pk: false,
un: false,
ai: false,
clen: 45,
np: null,
ns: null,
dtxp: '',
dtxs: '',
altered: 1,
uidt: UITypes.LastModifiedTime,
uip: '',
uicn: '',
system: true,
},
{
column_name: 'created_by',
title: 'nc_created_by',
dt: 'string',
dtx: 'specificType',
ct: 'string',
nrqd: true,
rqd: false,
ck: false,
pk: false,
un: false,
ai: false,
clen: 45,
np: null,
ns: null,
dtxp: '',
dtxs: '',
altered: 1,
uidt: UITypes.CreatedBy,
uip: '',
uicn: '',
system: true,
},
{
column_name: 'updated_by',
title: 'nc_updated_by',
dt: 'string',
dtx: 'specificType',
ct: 'string',
nrqd: true,
rqd: false,
ck: false,
pk: false,
un: false,
ai: false,
clen: 45,
np: null,
ns: null,
dtxp: '',
dtxs: '',
altered: 1,
uidt: UITypes.LastModifiedBy,
uip: '',
uicn: '',
system: true,
},
];
}
static getNewColumn(suffix) {
return {
column_name: 'title' + suffix,
dt: 'string',
dtx: 'specificType',
ct: 'string',
nrqd: true,
rqd: false,
ck: false,
pk: false,
un: false,
ai: false,
cdf: null,
clen: 45,
np: null,
ns: null,
dtxp: '',
dtxs: '',
altered: 1,
uidt: 'SingleLineText',
uip: '',
uicn: '',
};
}
static getDefaultLengthForDatatype(_type) {
return '';
}
static getDefaultLengthIsDisabled(type): any {
switch (type) {
case 'decimal':
return true;
case 'text':
return false;
}
}
static getDefaultValueForDatatype(type): any {
switch (type) {
case 'integer':
return 'eg : ' + 10;
case 'text':
return 'eg : hey';
case 'numeric':
return 'eg : ' + 10;
case 'real':
return 'eg : ' + 10.0;
case 'blob':
return 'eg : ' + 100;
}
}
static getDefaultScaleForDatatype(type): any {
switch (type) {
case 'integer':
case 'text':
case 'numeric':
case 'real':
case 'blob':
return ' ';
}
}
static colPropAIDisabled(col, columns) {
// console.log(col);
if (col.dt === 'integer') {
for (let i = 0; i < columns.length; ++i) {
if (columns[i].cn !== col.cn && columns[i].ai) {
return true;
}
}
return false;
} else {
return true;
}
}
static colPropUNDisabled(_col) {
return true;
}
static onCheckboxChangeAI(col) {
console.log(col);
if (
col.dt === 'int' ||
col.dt === 'bigint' ||
col.dt === 'smallint' ||
col.dt === 'tinyint'
) {
col.altered = col.altered || 2;
}
}
static showScale(_columnObj) {
return false;
}
static removeUnsigned(columns) {
for (let i = 0; i < columns.length; ++i) {
if (
columns[i].altered === 1 &&
!(
columns[i].dt === 'int' ||
columns[i].dt === 'bigint' ||
columns[i].dt === 'tinyint' ||
columns[i].dt === 'smallint' ||
columns[i].dt === 'mediumint'
)
) {
columns[i].un = false;
console.log('>> resetting unsigned value', columns[i].cn);
}
console.log(columns[i].cn);
}
}
/*static extractFunctionName(query) {
const reg =
/^\s*CREATE\s+(?:OR\s+REPLACE\s*)?\s*FUNCTION\s+(?:[\w\d_]+\.)?([\w_\d]+)/i;
const match = query.match(reg);
return match && match[1];
}
static extractProcedureName(query) {
const reg =
/^\s*CREATE\s+(?:OR\s+REPLACE\s*)?\s*PROCEDURE\s+(?:[\w\d_]+\.)?([\w_\d]+)/i;
const match = query.match(reg);
return match && match[1];
}*/
static columnEditable(_colObj) {
return true; // colObj.altered === 1;
}
/*static handleRawOutput(result, headers) {
console.log(result);
if (Array.isArray(result) && result[0]) {
const keys = Object.keys(result[0]);
// set headers before settings result
for (let i = 0; i < keys.length; i++) {
const text = keys[i];
headers.push({ text, value: text, sortable: false });
}
}
return result;
}
static splitQueries(query) {
/!***
* we are splitting based on semicolon
* there are mechanism to escape semicolon within single/double quotes(string)
*!/
return query.match(/\b("[^"]*;[^"]*"|'[^']*;[^']*'|[^;])*;/g);
}
/!**
* if sql statement is SELECT - it limits to a number
* @param args
* @returns {string|*}
*!/
sanitiseQuery(args) {
let q = args.query.trim().split(';');
if (q[0].startsWith('Select')) {
q = q[0] + ` LIMIT 0,${args.limit ? args.limit : 100};`;
} else if (q[0].startsWith('select')) {
q = q[0] + ` LIMIT 0,${args.limit ? args.limit : 100};`;
} else if (q[0].startsWith('SELECT')) {
q = q[0] + ` LIMIT 0,${args.limit ? args.limit : 100};`;
} else {
return args.query;
}
return q;
}
static getColumnsFromJson(json, tn) {
const columns = [];
try {
if (typeof json === 'object' && !Array.isArray(json)) {
const keys = Object.keys(json);
for (let i = 0; i < keys.length; ++i) {
const column = {
dp: null,
tn,
column_name: keys[i],
cno: keys[i],
np: null,
ns: null,
clen: null,
cop: 1,
pk: false,
nrqd: false,
rqd: false,
un: false,
ct: 'int(11) unsigned',
ai: false,
unique: false,
cdf: null,
cc: '',
csn: null,
dtx: 'specificType',
dtxp: null,
dtxs: 0,
altered: 1,
};
switch (typeof json[keys[i]]) {
case 'number':
if (Number.isInteger(json[keys[i]])) {
if (SqliteUi.isValidTimestamp(keys[i], json[keys[i]])) {
Object.assign(column, {
dt: 'timestamp',
});
} else {
Object.assign(column, {
dt: 'integer',
});
}
} else {
Object.assign(column, {
dt: 'real',
});
}
break;
case 'string':
// if (SqliteUi.isValidDate(json[keys[i]])) {
// Object.assign(column, {
// "dt": "datetime"
// });
// } else
if (json[keys[i]].length <= 255) {
Object.assign(column, {
dt: 'varchar',
});
} else {
Object.assign(column, {
dt: 'text',
});
}
break;
case 'boolean':
Object.assign(column, {
dt: 'integer',
});
break;
case 'object':
Object.assign(column, {
dt: 'text',
np: null,
dtxp: null,
});
break;
default:
break;
}
columns.push(column);
}
}
} catch (e) {
console.log('Error in getColumnsFromJson', e);
}
return columns;
}
static isValidTimestamp(key, value) {
if (typeof value !== 'number') {
return false;
}
return new Date(value).getTime() > 0 && /(?:_|(?=A))[aA]t$/.test(key);
}
static isValidDate(value) {
return new Date(value).getTime() > 0;
}*/
static onCheckboxChangeAU(col) {
console.log(col);
// if (1) {
col.altered = col.altered || 2;
// }
// if (!col.ai) {
// col.dtx = 'specificType'
// } else {
// col.dtx = ''
// }
}
static colPropAuDisabled(col) {
if (col.altered !== 1) {
return true;
}
switch (col.dt) {
case 'date':
case 'datetime':
case 'timestamp':
case 'time':
return false;
default:
return true;
}
}
static getAbstractType(col): any {
switch (col.dt?.replace(/\(\d+\)$/).toLowerCase()) {
case 'bigint':
case 'tinyint':
case 'int':
case 'smallint':
return 'integer';
case 'decimal':
case 'double':
case 'float':
return 'float';
case 'boolean':
return 'boolean';
case 'timestamp':
case 'timestamp_ntz':
return 'datetime';
case 'date':
return 'date';
case 'string':
return 'string';
}
return 'string';
}
static getUIType(col): any {
switch (this.getAbstractType(col)) {
case 'integer':
return 'Number';
case 'boolean':
return 'Checkbox';
case 'float':
return 'Decimal';
case 'date':
return 'Date';
case 'datetime':
return 'CreatedTime';
case 'time':
return 'Time';
case 'year':
return 'Year';
case 'string':
return 'SingleLineText';
case 'text':
return 'LongText';
case 'blob':
return 'Attachment';
case 'enum':
return 'SingleSelect';
case 'set':
return 'MultiSelect';
case 'json':
return 'LongText';
}
}
static getDataTypeForUiType(col: { uidt: UITypes; }, idType?: IDType) {
const colProp: any = {};
switch (col.uidt) {
case 'ID':
{
const isAutoIncId = idType === 'AI';
const isAutoGenId = idType === 'AG';
colProp.dt = isAutoGenId ? 'varchar' : 'integer';
colProp.pk = true;
colProp.un = isAutoIncId;
colProp.ai = isAutoIncId;
colProp.rqd = true;
colProp.meta = isAutoGenId ? { ag: 'nc' } : undefined;
}
break;
case 'ForeignKey':
colProp.dt = 'string';
break;
case 'SingleLineText':
colProp.dt = 'string';
break;
case 'LongText':
colProp.dt = 'string';
break;
case 'Attachment':
colProp.dt = 'string';
break;
case 'GeoData':
colProp.dt = 'string';
break;
case 'Checkbox':
colProp.dt = 'boolean';
break;
case 'MultiSelect':
colProp.dt = 'string';
break;
case 'SingleSelect':
colProp.dt = 'string';
break;
case 'Collaborator':
colProp.dt = 'string';
break;
case 'Date':
colProp.dt = 'date';
break;
case 'Year':
colProp.dt = 'number';
break;
case 'Time':
colProp.dt = 'string';
break;
case 'PhoneNumber':
colProp.dt = 'string';
colProp.validate = {
func: ['isMobilePhone'],
args: [''],
msg: ['Validation failed : isMobilePhone'],
};
break;
case 'Email':
colProp.dt = 'string';
colProp.validate = {
func: ['isEmail'],
args: [''],
msg: ['Validation failed : isEmail'],
};
break;
case 'URL':
colProp.dt = 'string';
colProp.validate = {
func: ['isURL'],
args: [''],
msg: ['Validation failed : isURL'],
};
break;
case 'Number':
colProp.dt = 'int';
break;
case 'Decimal':
colProp.dt = 'decimal';
break;
case 'Currency':
colProp.dt = 'double';
colProp.validate = {
func: ['isCurrency'],
args: [''],
msg: ['Validation failed : isCurrency'],
};
break;
case 'Percent':
colProp.dt = 'double';
break;
case 'Duration':
colProp.dt = 'decimal';
break;
case 'Rating':
colProp.dt = 'int';
colProp.cdf = '0';
break;
case 'Formula':
colProp.dt = 'string';
break;
case 'Rollup':
colProp.dt = 'string';
break;
case 'Count':
colProp.dt = 'int';
break;
case 'Lookup':
colProp.dt = 'string';
break;
case 'DateTime':
colProp.dt = 'datetime';
break;
case 'CreatedTime':
colProp.dt = 'datetime';
break;
case 'LastModifiedTime':
colProp.dt = 'datetime';
break;
case 'AutoNumber':
colProp.dt = 'int';
break;
case 'Barcode':
colProp.dt = 'string';
break;
case 'Button':
colProp.dt = 'string';
break;
case 'JSON':
colProp.dt = 'string';
break;
default:
colProp.dt = 'string';
break;
}
return colProp;
}
static getDataTypeListForUiType(col: { uidt: UITypes; }, idType?: IDType) {
switch (col.uidt) {
case 'ID':
if (idType === 'AG') {
return ['character', 'text', 'varchar'];
} else if (idType === 'AI') {
return [
'int',
'integer',
'tinyint',
'smallint',
'mediumint',
'bigint',
'int2',
'int8',
];
} else {
return dbTypes;
}
case 'ForeignKey':
return dbTypes;
case 'SingleLineText':
case 'LongText':
case 'Attachment':
case 'Collaborator':
case 'GeoData':
return ['string'];
case 'Checkbox':
return [
'boolean',
];
case 'MultiSelect':
return ['string'];
case 'SingleSelect':
return ['string'];
case 'Year':
return [
'int',
];
case 'Time':
return [
'string',
];
case 'PhoneNumber':
case 'Email':
return ['string'];
case 'URL':
return ['string'];
case 'Number':
return [
'int',
];
case 'Decimal':
return ['decimal', 'float', 'double'];
case 'Currency':
return [
'decimal',
];
case 'Percent':
return [
'decimal',
];
case 'Duration':
return [
'decimal',
];
case 'Rating':
return [
'int',
];
case 'Formula':
return ['string'];
case 'Rollup':
return ['string'];
case 'Count':
return [
'int',
];
case 'Lookup':
return ['string'];
case 'Date':
return ['date'];
case 'DateTime':
case 'CreatedTime':
case 'LastModifiedTime':
return ['datetime'];
case 'AutoNumber':
return [
'int',
];
case 'Barcode':
return ['string'];
case 'Geometry':
return ['string'];
case 'JSON':
return ['string'];
case 'Button':
default:
return dbTypes;
}
}
static getUnsupportedFnList() {
return [
'LOG',
'EXP',
'POWER',
'SQRT',
'XOR',
'REGEX_MATCH',
'REGEX_EXTRACT',
'REGEX_REPLACE',
'VALUE',
'COUNTA',
'COUNT',
'ROUNDDOWN',
'ROUNDUP',
'DATESTR',
'DAY',
'MONTH',
'HOUR',
];
}
}

5
packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts

@ -7,6 +7,7 @@ import { OracleUi } from './OracleUi';
import { PgUi } from './PgUi';
import { SqliteUi } from './SqliteUi';
import { SnowflakeUi } from './SnowflakeUi';
import { DatabricksUi } from './DatabricksUi';
// import {YugabyteUi} from "./YugabyteUi";
// import {TidbUi} from "./TidbUi";
@ -48,6 +49,10 @@ export class SqlUiFactory {
return SnowflakeUi;
}
if (connectionConfig.client === 'databricks') {
return DatabricksUi;
}
throw new Error('Database not supported');
}
}

18
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -2770,16 +2770,16 @@ class BaseModelSqlv2 {
{ raw: true, first: true },
)
)?.__nc_ai_id;
} else if (this.isSnowflake) {
} else if (this.isSnowflake || this.isDatabricks) {
id = (
await this.execAndParse(
this.dbDriver(this.tnPath).max(ai.column_name, {
as: 'id',
as: '__nc_ai_id',
}),
null,
{ raw: true, first: true },
)
).id;
).__nc_ai_id;
}
response = await this.readByPk(
this.extractCompositePK({ rowId: id, insertObj, ag }),
@ -3054,6 +3054,10 @@ class BaseModelSqlv2 {
return this.clientType === 'snowflake';
}
get isDatabricks() {
return this.clientType === 'databricks';
}
get clientType() {
return this.dbDriver.clientType();
}
@ -3161,16 +3165,16 @@ class BaseModelSqlv2 {
},
)
)?.__nc_ai_id;
} else if (this.isSnowflake) {
} else if (this.isSnowflake || this.isDatabricks) {
rowId = (
await this.execAndParse(
this.dbDriver(this.tnPath).max(ai.column_name, {
as: 'id',
as: '__nc_ai_id',
}),
null,
{ raw: true, first: true },
)
)?.id;
)?.__nc_ai_id;
}
// response = await this.readByPk(
// id,
@ -4477,7 +4481,7 @@ class BaseModelSqlv2 {
const vTn = this.getTnPath(vTable);
if (this.isSnowflake) {
if (this.isSnowflake || this.isDatabricks) {
const parentPK = this.dbDriver(parentTn)
.select(parentColumn.column_name)
.where(_wherePk(parentTable.primaryKeys, childId))

26
packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts

@ -833,6 +833,19 @@ async function _formulaQueryBuilder(
} else {
return fn(pt.arguments[0], a, prevBinaryOp);
}
} else if (knex.clientType() === 'databricks') {
const res = await mapFunctionName({
pt,
knex,
alias,
a,
aliasToCol: aliasToColumn,
fn,
colAlias,
prevBinaryOp,
model,
});
if (res) return res;
}
break;
case 'URL':
@ -921,6 +934,19 @@ async function _formulaQueryBuilder(
if (typeof builder === 'function') {
return { builder: knex.raw(`??${colAlias}`, builder(pt.fnName)) };
}
if (
knex.clientType() === 'databricks' &&
builder.toQuery().endsWith(')')
) {
// limit 1 for subquery
return {
builder: knex.raw(
`${builder.toQuery().replace(/\)$/, '')} LIMIT 1)${colAlias}`,
),
};
}
return { builder: knex.raw(`??${colAlias}`, [builder || pt.name]) };
} else if (pt.type === 'BinaryExpression') {
// treat `&` as shortcut for concat

44
packages/nocodb/src/db/functionMappings/databricks.ts

@ -0,0 +1,44 @@
import commonFns from './commonFns';
import type { MapFnArgs } from '../mapFunctionName';
const databricks = {
...commonFns,
AND: async (args: MapFnArgs) => {
return {
builder: args.knex.raw(
`CASE WHEN ${args.knex
.raw(
`${(
await Promise.all(
args.pt.arguments.map(async (ar) =>
(await args.fn(ar, '', 'AND')).builder.toQuery(),
),
)
).join(' AND ')}`,
)
.wrap('(', ')')
.toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}`,
),
};
},
OR: async (args: MapFnArgs) => {
return {
builder: args.knex.raw(
`CASE WHEN ${args.knex
.raw(
`${(
await Promise.all(
args.pt.arguments.map(async (ar) =>
(await args.fn(ar, '', 'OR')).builder.toQuery(),
),
)
).join(' OR ')}`,
)
.wrap('(', ')')
.toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}`,
),
};
},
};
export default databricks;

4
packages/nocodb/src/db/mapFunctionName.ts

@ -5,6 +5,7 @@ import mssql from '~/db/functionMappings/mssql';
import mysql from '~/db/functionMappings/mysql';
import pg from '~/db/functionMappings/pg';
import sqlite from '~/db/functionMappings/sqlite';
import databricks from '~/db/functionMappings/databricks';
export interface MapFnArgs {
pt: any;
@ -42,6 +43,9 @@ const mapFunctionName = async (args: MapFnArgs): Promise<any> => {
case 'sqlite3':
val = sqlite[name] || name;
break;
case 'databricks':
val = databricks[name] || name;
break;
}
if (typeof val === 'function') {

6
packages/nocodb/src/db/sql-client/lib/SqlClientFactory.ts

@ -1,5 +1,6 @@
import fs from 'fs';
import { promisify } from 'util';
import { DatabricksClient } from 'nc-help';
import MySqlClient from '~/db/sql-client/lib/mysql/MysqlClient';
import MssqlClient from '~/db/sql-client/lib/mssql/MssqlClient';
import OracleClient from '~/db/sql-client/lib/oracle/OracleClient';
@ -8,6 +9,7 @@ import PgClient from '~/db/sql-client/lib/pg/PgClient';
import YugabyteClient from '~/db/sql-client/lib/pg/YugabyteClient';
import TidbClient from '~/db/sql-client/lib/mysql/TidbClient';
import VitessClient from '~/db/sql-client/lib/mysql/VitessClient';
import DbrClient from '~/db/sql-client/lib/databricks/DatabricksClient';
export class SqlClientFactory {
static create(connectionConfig) {
@ -33,6 +35,10 @@ export class SqlClientFactory {
if (connectionConfig.meta.dbtype === 'yugabyte')
return new YugabyteClient(connectionConfig);
return new PgClient(connectionConfig);
} else if (connectionConfig.client === 'databricks') {
connectionConfig.client = DatabricksClient;
connectionConfig.pool = { min: 0, max: 1 };
return new DbrClient(connectionConfig);
}
throw new Error('Database not supported');

2621
packages/nocodb/src/db/sql-client/lib/databricks/DatabricksClient.ts

File diff suppressed because it is too large Load Diff

3
packages/nocodb/src/db/sql-client/lib/databricks/databricks.queries.ts

@ -0,0 +1,3 @@
const queries = {};
export default queries;

509
packages/nocodb/src/db/sql-mgr/code/models/xc/ModelXcMetaDatabricks.ts

@ -0,0 +1,509 @@
import BaseModelXcMeta from './BaseModelXcMeta';
class ModelXcMetaDatabricks extends BaseModelXcMeta {
/**
* @param dir
* @param filename
* @param ctx
* @param ctx.tn
* @param ctx.columns
* @param ctx.relations
*/
constructor({ dir, filename, ctx }) {
super({ dir, filename, ctx });
}
/**
* Prepare variables used in code template
*/
prepare() {
const data: any = {};
/* run of simple variable */
data.tn = this.ctx.tn;
data.dbType = this.ctx.dbType;
/* for complex code provide a func and args - do derivation within the func cbk */
data.columns = {
func: this._renderXcColumns.bind(this),
args: {
tn: this.ctx.tn,
columns: this.ctx.columns,
relations: this.ctx.relations,
},
};
/* for complex code provide a func and args - do derivation within the func cbk */
data.hasMany = {
func: this.renderXcHasMany.bind(this),
args: {
tn: this.ctx.tn,
columns: this.ctx.columns,
hasMany: this.ctx.hasMany,
},
};
/* for complex code provide a func and args - do derivation within the func cbk */
data.belongsTo = {
func: this.renderXcBelongsTo.bind(this),
args: {
tn: this.ctx.tn,
columns: this.ctx.columns,
belongsTo: this.ctx.belongsTo,
},
};
return data;
}
/**
*
* @param args
* @param args.columns
* @param args.relations
* @returns {string}
* @private
*/
_renderXcColumns(args) {
let str = '[\r\n';
for (let i = 0; i < args.columns.length; ++i) {
str += `{\r\n`;
str += `cn: '${args.columns[i].cn}',\r\n`;
str += `type: '${this._getAbstractType(args.columns[i])}',\r\n`;
str += `dt: '${args.columns[i].dt}',\r\n`;
if (args.columns[i].rqd) str += `rqd: ${args.columns[i].rqd},\r\n`;
if (args.columns[i].cdf) {
str += `default: "${args.columns[i].cdf}",\r\n`;
str += `columnDefault: "${args.columns[i].cdf}",\r\n`;
}
if (args.columns[i].un) str += `un: ${args.columns[i].un},\r\n`;
if (args.columns[i].pk) str += `pk: ${args.columns[i].pk},\r\n`;
if (args.columns[i].ai) str += `ai: ${args.columns[i].ai},\r\n`;
if (args.columns[i].dtxp) str += `dtxp: "${args.columns[i].dtxp}",\r\n`;
if (args.columns[i].dtxs) str += `dtxs: ${args.columns[i].dtxs},\r\n`;
str += `validate: {
func: [],
args: [],
msg: []
},`;
str += `},\r\n`;
}
str += ']\r\n';
return str;
}
_getAbstractType(column) {
return this.getAbstractType(column);
}
getUIDataType(col): any {
const dt = col.dt.toLowerCase();
switch (dt) {
case 'bigint':
case 'tinyint':
case 'int':
case 'smallint':
return 'Number';
case 'decimal':
case 'double':
case 'float':
return 'Decimal';
case 'boolean':
return 'Checkbox';
case 'timestamp':
case 'timestamp_ntz':
return 'DateTime';
case 'date':
return 'Date';
case 'string':
return 'LongText';
case 'interval':
case 'void':
case 'binary':
default:
return 'SpecificDBType';
}
}
getAbstractType(col): any {
const dt = col.dt.toLowerCase();
switch (dt) {
case 'bigint':
case 'tinyint':
case 'decimal':
case 'double':
case 'float':
case 'int':
case 'smallint':
return 'integer';
case 'binary':
return dt;
case 'boolean':
return 'boolean';
case 'interval':
case 'void':
return dt;
case 'timestamp':
case 'timestamp_ntz':
return 'datetime';
case 'date':
return 'date';
case 'string':
return 'string';
}
}
_sequelizeGetType(column) {
let str = '';
switch (column.dt) {
case 'int':
str += `DataTypes.INTEGER(${column.dtxp})`;
if (column.un) str += `.UNSIGNED`;
break;
case 'tinyint':
str += `DataTypes.INTEGER(${column.dtxp})`;
if (column.un) str += `.UNSIGNED`;
break;
case 'smallint':
str += `DataTypes.INTEGER(${column.dtxp})`;
if (column.un) str += `.UNSIGNED`;
break;
case 'mediumint':
str += `DataTypes.INTEGER(${column.dtxp})`;
if (column.un) str += `.UNSIGNED`;
break;
case 'bigint':
str += `DataTypes.BIGINT`;
if (column.un) str += `.UNSIGNED`;
break;
case 'float':
str += `DataTypes.FLOAT`;
break;
case 'decimal':
str += `DataTypes.DECIMAL`;
break;
case 'double':
str += `"DOUBLE(${column.dtxp},${column.ns})"`;
break;
case 'real':
str += `DataTypes.FLOAT`;
break;
case 'bit':
str += `DataTypes.BOOLEAN`;
break;
case 'boolean':
str += `DataTypes.STRING(45)`;
break;
case 'serial':
str += `DataTypes.BIGINT`;
break;
case 'date':
str += `DataTypes.DATEONLY`;
break;
case 'datetime':
str += `DataTypes.DATE`;
break;
case 'timestamp':
str += `DataTypes.DATE`;
break;
case 'time':
str += `DataTypes.TIME`;
break;
case 'year':
str += `"YEAR"`;
break;
case 'char':
str += `DataTypes.CHAR(${column.dtxp})`;
break;
case 'varchar':
str += `DataTypes.STRING(${column.dtxp})`;
break;
case 'nchar':
str += `DataTypes.CHAR(${column.dtxp})`;
break;
case 'text':
str += `DataTypes.TEXT`;
break;
case 'tinytext':
str += `DataTypes.TEXT`;
break;
case 'mediumtext':
str += `DataTypes.TEXT`;
break;
case 'longtext':
str += `DataTypes.TEXT`;
break;
case 'binary':
str += `"BINARY(${column.dtxp})"`;
break;
case 'varbinary':
str += `"VARBINARY(${column.dtxp})"`;
break;
case 'blob':
str += `"BLOB"`;
break;
case 'tinyblob':
str += `"TINYBLOB"`;
break;
case 'mediumblob':
str += `"MEDIUMBLOB"`;
break;
case 'longblob':
str += `"LONGBLOB"`;
break;
case 'enum':
str += `DataTypes.ENUM(${column.dtxp})`;
break;
case 'set':
str += `"SET(${column.dtxp})"`;
break;
case 'geometry':
str += `DataTypes.GEOMETRY`;
break;
case 'point':
str += `"POINT"`;
break;
case 'linestring':
str += `"LINESTRING"`;
break;
case 'polygon':
str += `"POLYGON"`;
break;
case 'multipoint':
str += `"MULTIPOINT"`;
break;
case 'multilinestring':
str += `"MULTILINESTRING"`;
break;
case 'multipolygon':
str += `"MULTIPOLYGON"`;
break;
case 'json':
str += `DataTypes.JSON`;
break;
default:
str += `"${column.dt}"`;
break;
}
return str;
}
_sequelizeGetDefault(column) {
let str = '';
switch (column.dt) {
case 'int':
str += `'${column.cdf}'`;
break;
case 'tinyint':
str += `'${column.cdf}'`;
break;
case 'smallint':
str += `'${column.cdf}'`;
break;
case 'mediumint':
str += `'${column.cdf}'`;
break;
case 'bigint':
str += `'${column.cdf}'`;
break;
case 'float':
str += `'${column.cdf}'`;
break;
case 'decimal':
str += `'${column.cdf}'`;
break;
case 'double':
str += `'${column.cdf}'`;
break;
case 'real':
str += `'${column.cdf}'`;
break;
case 'bit':
str += column.cdf ? column.cdf.split('b')[1] : column.cdf;
break;
case 'boolean':
str += column.cdf;
break;
case 'serial':
str += column.cdf;
break;
case 'date':
str += `sequelize.literal('${column.cdf_sequelize}')`;
break;
case 'datetime':
str += `sequelize.literal('${column.cdf_sequelize}')`;
break;
case 'timestamp':
str += `sequelize.literal('${column.cdf_sequelize}')`;
break;
case 'time':
str += `'${column.cdf}'`;
break;
case 'year':
str += `'${column.cdf}'`;
break;
case 'char':
str += `'${column.cdf}'`;
break;
case 'varchar':
str += `'${column.cdf}'`;
break;
case 'nchar':
str += `'${column.cdf}'`;
break;
case 'text':
str += column.cdf;
break;
case 'tinytext':
str += column.cdf;
break;
case 'mediumtext':
str += column.cdf;
break;
case 'longtext':
str += column.cdf;
break;
case 'binary':
str += column.cdf;
break;
case 'varbinary':
str += column.cdf;
break;
case 'blob':
str += column.cdf;
break;
case 'tinyblob':
str += column.cdf;
break;
case 'mediumblob':
str += column.cdf;
break;
case 'longblob':
str += column.cdf;
break;
case 'enum':
str += `'${column.cdf}'`;
break;
case 'set':
str += `'${column.cdf}'`;
break;
case 'geometry':
str += `'${column.cdf}'`;
break;
case 'point':
str += `'${column.cdf}'`;
break;
case 'linestring':
str += `'${column.cdf}'`;
break;
case 'polygon':
str += `'${column.cdf}'`;
break;
case 'multipoint':
str += `'${column.cdf}'`;
break;
case 'multilinestring':
str += `'${column.cdf}'`;
break;
case 'multipolygon':
str += `'${column.cdf}'`;
break;
case 'json':
str += column.cdf;
break;
}
return str;
}
/* getXcColumnsObject(args) {
const columnsArr = [];
for (const column of args.columns) {
const columnObj = {
validate: {
func: [],
args: [],
msg: []
},
cn: column.cn,
_cn: column._cn || column.cn,
type: this._getAbstractType(column),
dt: column.dt,
uidt: column.uidt || this._getUIDataType(column),
uip: column.uip,
uicn: column.uicn,
...column
};
if (column.rqd) {
columnObj.rqd = column.rqd;
}
if (column.cdf) {
columnObj.default = column.cdf;
columnObj.columnDefault = column.cdf;
}
if (column.un) {
columnObj.un = column.un;
}
if (column.pk) {
columnObj.pk = column.pk;
}
if (column.ai) {
columnObj.ai = column.ai;
}
if (column.dtxp) {
columnObj.dtxp = column.dtxp;
}
if (column.dtxs) {
columnObj.dtxs = column.dtxs;
}
columnsArr.push(columnObj);
}
this.mapDefaultDisplayValue(columnsArr);
return columnsArr;
}*/
/* getObject() {
return {
tn: this.ctx.tn,
_tn: this.ctx._tn,
columns: this.getXcColumnsObject(this.ctx),
pks: [],
hasMany: this.ctx.hasMany,
belongsTo: this.ctx.belongsTo,
dbType: this.ctx.dbType,
type: this.ctx.type,
}
}*/
}
export default ModelXcMetaDatabricks;

3
packages/nocodb/src/db/sql-mgr/code/models/xc/ModelXcMetaFactory.ts

@ -4,6 +4,7 @@ import ModelXcMetaOracle from './ModelXcMetaOracle';
import ModelXcMetaPg from './ModelXcMetaPg';
import ModelXcMetaSqlite from './ModelXcMetaSqlite';
import ModelXcMetaSnowflake from './ModelXcMetaSnowflake';
import ModelXcMetaDatabricks from './ModelXcMetaDatabricks';
import type BaseModelXcMeta from './BaseModelXcMeta';
class ModelXcMetaFactory {
@ -23,6 +24,8 @@ class ModelXcMetaFactory {
return new ModelXcMetaOracle(args);
} else if (connectionConfig.client === 'snowflake') {
return new ModelXcMetaSnowflake(args);
} else if (connectionConfig.client === 'databricks') {
return new ModelXcMetaDatabricks(args);
}
throw new Error('Database not supported');

8
packages/nocodb/src/helpers/populateMeta.ts

@ -334,6 +334,14 @@ export async function populateMeta(
let colOrder = 1;
for (const column of columns) {
if (source.type === 'databricks') {
if (column.pk && !column.cdf) {
column.meta = {
ag: 'nc',
};
}
}
await Column.insert({
uidt: column.uidt || getColumnUiType(source, column),
fk_model_id: models2[table.tn].id,

9
packages/nocodb/src/schema/swagger-v2.json

@ -9737,7 +9737,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"example": "mysql2",
"type": "string"
@ -11989,7 +11990,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"example": "mysql2",
"type": "string"
@ -12153,7 +12155,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"type": "string"
}

12
packages/nocodb/src/schema/swagger.json

@ -14832,7 +14832,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"example": "mysql2",
"type": "string"
@ -14977,7 +14978,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"example": "mysql2",
"type": "string"
@ -17900,7 +17902,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"example": "mysql2",
"type": "string"
@ -18064,7 +18067,8 @@
"oracledb",
"pg",
"snowflake",
"sqlite3"
"sqlite3",
"databricks"
],
"type": "string"
}

12
packages/nocodb/src/services/meta-diffs.service.ts

@ -141,6 +141,8 @@ export class MetaDiffsService {
const changes: Array<MetaDiff> = [];
const virtualRelationColumns: Column<LinkToAnotherRecordColumn>[] = [];
const isDatabricks = sqlClient.knex.clientType() === 'databricks';
// @ts-ignore
const tableList: Array<{ tn: string }> = (
await sqlClient.tableList({ schema: source.getConfig()?.schema })
@ -178,7 +180,10 @@ export class MetaDiffsService {
if (table.tn === 'nc_evolutions') continue;
const oldMetaIdx = oldTableMetas.findIndex(
(m) => m.table_name === table.tn,
(m) =>
m.table_name === table.tn ||
(isDatabricks &&
m.table_name.toLowerCase() === table.tn.toLowerCase()),
);
// new table
@ -223,7 +228,10 @@ export class MetaDiffsService {
for (const column of colListRef[table.tn]) {
const oldColIdx = oldMeta.columns.findIndex(
(c) => c.column_name === column.cn,
(c) =>
c.column_name === column.cn ||
(isDatabricks &&
c.column_name.toLowerCase() === column.cn.toLowerCase()),
);
// new table

4
packages/nocodb/src/services/tables.service.ts

@ -82,6 +82,8 @@ export class TablesService {
}
}
param.table.table_name = param.table.table_name.replace(/ /g, '_');
param.table.table_name = DOMPurify.sanitize(param.table.table_name);
// validate table name
@ -519,7 +521,7 @@ export class TablesService {
}
tableCreatePayLoad.table_name = DOMPurify.sanitize(
tableCreatePayLoad.table_name,
tableCreatePayLoad.table_name.replace(/ /g, '_'),
);
// validate table name

1
packages/nocodb/src/utils/globals.ts

@ -256,4 +256,5 @@ export const DB_TYPES = <const>[
'snowflake',
'oracledb',
'pg',
'databricks',
];

1
packages/nocodb/src/utils/nc-config/constants.ts

@ -81,4 +81,5 @@ export enum DriverClient {
PG = 'pg',
SQLITE = 'sqlite3',
SNOWFLAKE = 'snowflake',
DATABRICKS = 'databricks',
}

Loading…
Cancel
Save