mirror of https://github.com/nocodb/nocodb
Browse Source
* chore: preps Signed-off-by: mertmit <mertmit99@gmail.com> * test: fix unit Signed-off-by: mertmit <mertmit99@gmail.com> --------- Signed-off-by: mertmit <mertmit99@gmail.com>pull/9999/head
Mert E.
3 days ago
committed by
GitHub
72 changed files with 1636 additions and 278 deletions
@ -0,0 +1,29 @@
|
||||
import type { Knex } from 'knex'; |
||||
import { MetaTable } from '~/utils/globals'; |
||||
|
||||
const up = async (knex: Knex) => { |
||||
await knex.schema.createTable(MetaTable.COL_LONG_TEXT, (table) => { |
||||
table.string('id', 20).primary(); |
||||
table.string('fk_workspace_id', 20); |
||||
table.string('base_id', 20); |
||||
table.string('fk_model_id', 20); |
||||
table.string('fk_column_id', 20); |
||||
|
||||
table.string('fk_integration_id', 20); |
||||
table.string('model', 255); |
||||
|
||||
table.text('prompt'); |
||||
table.text('prompt_raw'); |
||||
table.text('error'); |
||||
|
||||
table.timestamps(true, true); |
||||
|
||||
table.index('fk_column_id'); |
||||
}); |
||||
}; |
||||
|
||||
const down = async (knex: Knex) => { |
||||
await knex.schema.dropTable(MetaTable.COL_LONG_TEXT); |
||||
}; |
||||
|
||||
export { up, down }; |
||||
@ -0,0 +1,48 @@
|
||||
import type { NcContext } from '~/interface/config'; |
||||
import Noco from '~/Noco'; |
||||
import LongTextColumn from '~/models/LongTextColumn'; |
||||
|
||||
export default class AIColumn extends LongTextColumn { |
||||
id: string; |
||||
|
||||
fk_integration_id: string; |
||||
model: string; |
||||
prompt: string; |
||||
prompt_raw: string; |
||||
error?: string; |
||||
|
||||
public static castType(data: AIColumn): AIColumn { |
||||
return data && new AIColumn(data); |
||||
} |
||||
|
||||
public static async insert( |
||||
context: NcContext, |
||||
aiColumn: Partial<AIColumn> & { |
||||
fk_model_id: string; |
||||
fk_column_id: string; |
||||
}, |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
return this._insert( |
||||
context, |
||||
aiColumn, |
||||
['fk_integration_id', 'model', 'prompt', 'prompt_raw', 'error'], |
||||
ncMeta, |
||||
); |
||||
} |
||||
|
||||
public static async update( |
||||
context: NcContext, |
||||
columnId: string, |
||||
aiColumn: Partial<AIColumn>, |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
return this._update( |
||||
context, |
||||
columnId, |
||||
aiColumn, |
||||
['fk_integration_id', 'model', 'prompt', 'prompt_raw', 'error'], |
||||
ncMeta, |
||||
); |
||||
} |
||||
} |
||||
@ -0,0 +1,113 @@
|
||||
import type { NcContext } from '~/interface/config'; |
||||
import Noco from '~/Noco'; |
||||
import NocoCache from '~/cache/NocoCache'; |
||||
import { extractProps } from '~/helpers/extractProps'; |
||||
import { CacheGetType, CacheScope, MetaTable } from '~/utils/globals'; |
||||
import { Column } from '~/models/index'; |
||||
import { NcError } from '~/helpers/catchError'; |
||||
|
||||
export default abstract class LongTextColumn { |
||||
id: string; |
||||
|
||||
fk_workspace_id?: string; |
||||
base_id: string; |
||||
fk_model_id: string; |
||||
fk_column_id: string; |
||||
|
||||
constructor(data: Partial<LongTextColumn>) { |
||||
Object.assign(this, data); |
||||
} |
||||
|
||||
public static castType(data: LongTextColumn): LongTextColumn { |
||||
return data; |
||||
} |
||||
|
||||
protected static async _insert( |
||||
context: NcContext, |
||||
longTextColumn: Partial<LongTextColumn> & { |
||||
fk_model_id: string; |
||||
fk_column_id: string; |
||||
}, |
||||
props: string[], |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
const insertObj = extractProps(longTextColumn, [ |
||||
'fk_workspace_id', |
||||
'base_id', |
||||
'fk_model_id', |
||||
'fk_column_id', |
||||
...(props || []), |
||||
]); |
||||
|
||||
const column = await Column.get( |
||||
context, |
||||
{ |
||||
colId: insertObj.fk_column_id, |
||||
}, |
||||
ncMeta, |
||||
); |
||||
|
||||
if (!column) { |
||||
NcError.fieldNotFound(insertObj.fk_column_id); |
||||
} |
||||
|
||||
await ncMeta.metaInsert2( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.COL_LONG_TEXT, |
||||
insertObj, |
||||
); |
||||
|
||||
return this.read(context, longTextColumn.fk_column_id, ncMeta); |
||||
} |
||||
|
||||
public static async read( |
||||
context: NcContext, |
||||
columnId: string, |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
let column = |
||||
columnId && |
||||
(await NocoCache.get( |
||||
`${CacheScope.COL_LONG_TEXT}:${columnId}`, |
||||
CacheGetType.TYPE_OBJECT, |
||||
)); |
||||
if (!column) { |
||||
column = await ncMeta.metaGet2( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.COL_LONG_TEXT, |
||||
{ fk_column_id: columnId }, |
||||
); |
||||
await NocoCache.set(`${CacheScope.COL_LONG_TEXT}:${columnId}`, column); |
||||
} |
||||
|
||||
return column ? this.castType(column) : null; |
||||
} |
||||
|
||||
protected static async _update( |
||||
context: NcContext, |
||||
columnId: string, |
||||
longTextColumn: Partial<LongTextColumn>, |
||||
props: string[], |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
const updateObj = extractProps(longTextColumn, [...(props || [])]); |
||||
|
||||
// set meta
|
||||
await ncMeta.metaUpdate( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.COL_LONG_TEXT, |
||||
updateObj, |
||||
{ |
||||
fk_column_id: columnId, |
||||
}, |
||||
); |
||||
|
||||
await NocoCache.update( |
||||
`${CacheScope.COL_LONG_TEXT}:${columnId}`, |
||||
updateObj, |
||||
); |
||||
} |
||||
} |
||||
@ -0,0 +1,150 @@
|
||||
import type { AIRecordType } from 'nocodb-sdk'; |
||||
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; |
||||
import type KnexClient from '~/db/sql-client/lib/KnexClient'; |
||||
import type { Column, Model, Source, User } from '~/models'; |
||||
|
||||
export const convertAIRecordTypeToValue = async (args: { |
||||
source: Source; |
||||
table: Model; |
||||
column: Column; |
||||
baseModel: BaseModelSqlv2; |
||||
sqlClient: KnexClient; |
||||
}) => { |
||||
const { source, table, column, baseModel, sqlClient } = args; |
||||
|
||||
if (source.type === 'pg') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = TRIM('"' FROM (??::jsonb->>'value')) |
||||
WHERE ?? ~ '^\\s*\\{.*\\}\\s*$' AND jsonb_typeof(??::jsonb) = 'object' AND (??::jsonb->'value') IS NOT NULL;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} else if (source.type === 'mysql' || source.type === 'mysql2') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = JSON_UNQUOTE(JSON_EXTRACT(??, '$.value')) |
||||
WHERE JSON_VALID(??) AND JSON_TYPE(??) = 'OBJECT' AND JSON_EXTRACT(??, '$.value') IS NOT NULL;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} else if (source.type === 'sqlite3') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = json_extract(??, '$.value') |
||||
WHERE json_valid(??) AND json_extract(??, '$.value') IS NOT NULL;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} else if (source.type === 'mssql') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = JSON_VALUE(??, '$.value') |
||||
WHERE ISJSON(??) = 1 AND JSON_VALUE(??, '$.value') IS NOT NULL;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} |
||||
}; |
||||
|
||||
export const convertValueToAIRecordType = async (args: { |
||||
source: Source; |
||||
table: Model; |
||||
column: Column; |
||||
baseModel: BaseModelSqlv2; |
||||
sqlClient: KnexClient; |
||||
user: User; |
||||
}) => { |
||||
const { source, table, column, baseModel, sqlClient, user } = args; |
||||
|
||||
const commonRecord: Omit<AIRecordType, 'value'> = { |
||||
lastModifiedBy: user.id, |
||||
lastModifiedTime: baseModel.now(), |
||||
isStale: true, |
||||
}; |
||||
|
||||
// update every record with json which holds old value in value prop & commonRecord props in lastModifiedBy, lastModifiedTime, isStale
|
||||
if (source.type === 'pg') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = jsonb_build_object('value', ??, 'lastModifiedBy', ?::text, 'lastModifiedTime', ?::text, 'isStale', ?::boolean) |
||||
WHERE ?? is not null;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
commonRecord.lastModifiedBy.toString(), |
||||
commonRecord.lastModifiedTime.toString(), |
||||
commonRecord.isStale, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} else if (source.type === 'mysql' || source.type === 'mysql2') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = JSON_OBJECT('value', ??, 'lastModifiedBy', ?, 'lastModifiedTime', ?, 'isStale', ?) |
||||
WHERE ?? is not null;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
commonRecord.lastModifiedBy.toString(), |
||||
commonRecord.lastModifiedTime.toString(), |
||||
commonRecord.isStale, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} else if (source.type === 'sqlite3') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = json_object('value', ??, 'lastModifiedBy', ?, 'lastModifiedTime', ?, 'isStale', ?) |
||||
WHERE ?? is not null;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
commonRecord.lastModifiedBy.toString(), |
||||
commonRecord.lastModifiedTime.toString(), |
||||
commonRecord.isStale, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} else if (source.type === 'mssql') { |
||||
await sqlClient.raw( |
||||
`UPDATE ??
|
||||
SET ?? = JSON_QUERY('{"value":' + ?? + ',"lastModifiedBy":' + ? + ',"lastModifiedTime":' + ? + ',"isStale":' + ? + '}') |
||||
WHERE ?? is not null;`,
|
||||
[ |
||||
baseModel.getTnPath(table.table_name), |
||||
column.column_name, |
||||
column.column_name, |
||||
commonRecord.lastModifiedBy.toString(), |
||||
commonRecord.lastModifiedTime.toString(), |
||||
commonRecord.isStale, |
||||
column.column_name, |
||||
], |
||||
); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue