github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge branch 'develop' into nc-fix/swagger-base-pw

pull/9543/head
Raju Udava 2 months ago
parent
6efdd79d4b bdd6942359
commit
d1ccb70bc4
19 changed files with 550 additions and 104 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 31
      docker-compose/1_Auto_Upstall/noco.sh
  2. 46
      packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md
  3. 2
      packages/nocodb/src/controllers/filters.controller.ts
  4. 8
      packages/nocodb/src/db/BaseModelSqlv2.ts
  5. 330
      packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts
  6. 57
      packages/nocodb/src/helpers/initBaseBehaviour.ts
  7. 4
      packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
  8. 16
      packages/nocodb/src/meta/migrations/v2/nc_064_pg_minimal_dbs.ts
  9. 13
      packages/nocodb/src/models/Model.ts
  10. 17
      packages/nocodb/src/models/Source.ts
  11. 7
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  12. 1
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
  13. 4
      packages/nocodb/src/providers/init-meta-service.provider.ts
  14. 72
      packages/nocodb/src/services/bases.service.ts
  15. 7
      packages/nocodb/src/services/data-alias-nested.service.ts
  16. 2
      packages/nocodb/src/services/data-table.service.ts
  17. 23
      packages/nocodb/src/services/datas.service.ts
  18. 10
      packages/nocodb/src/services/public-datas.service.ts
  19. 4
      packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

31
docker-compose/1_Auto_Upstall/noco.sh
Unescape View File

@ -496,11 +496,20 @@ generate_credentials() {
create_docker_compose_file() {
image="nocodb/nocodb:latest"
if [ "${CONFIG_EDITION}" = "EE" ] || [ "${CONFIG_EDITION}" = "ee" ]; then
image="nocodb/nocodb-ee:latest"
else
image="nocodb/nocodb:latest"
fi
if [ "${CONFIG_EDITION}" = "EE" ] || [ "${CONFIG_EDITION}" = "ee" ]; then
image="nocodb/ee:latest"
fi
# for easier string interpolation
if [ "${CONFIG_REDIS_ENABLED}" = "Y" ]; then
gen_redis=1
fi
if [ "${CONFIG_MINIO_ENABLED}" = "Y" ]; then
gen_minio=1
fi
local compose_file="docker-compose.yml"
@ -514,8 +523,8 @@ services:
replicas: ${CONFIG_NUM_INSTANCES}
depends_on:
- db
${CONFIG_REDIS_ENABLED:+- redis}
${CONFIG_MINIO_ENABLED:+- minio}
${gen_redis:+- redis}
${gen_minio:+- minio}
restart: unless-stopped
volumes:
- ./nocodb:/usr/app/data
@ -636,7 +645,6 @@ EOF
- "traefik.http.routers.minio.rule=Host(\`${CONFIG_MINIO_DOMAIN_NAME}\`)"
EOF
# If minio SSL is enabled, set the entry point to websecure
fi
if [ "$CONFIG_MINIO_SSL_ENABLED" = "Y" ]; then
cat >> "$compose_file" <<EOF
- "traefik.http.routers.minio.entrypoints=websecure"
@ -660,6 +668,7 @@ EOF
- nocodb-network
EOF
fi
if [ "${CONFIG_WATCHTOWER_ENABLED}" = "Y" ]; then
cat >> "$compose_file" <<EOF
watchtower:
@ -674,10 +683,14 @@ EOF
EOF
fi
cat >> "$compose_file" <<EOF
if [ "$CONFIG_REDIS_ENABLED" = "Y" ]; then
cat >> "$compose_file" <<EOF
volumes:
${CONFIG_REDIS_ENABLED:+redis:}
redis:
EOF
fi
cat >> "$compose_file" <<EOF
networks:
nocodb-network:
driver: bridge

46
packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md
Unescape View File

@ -75,24 +75,34 @@ For production use cases, it is crucial to set all environment variables marked
| `NC_REDIS_URL` | Yes | Specifies the Redis URL used for caching. <br></br> Eg: `redis://:authpassword@127.0.0.1:6380/4` | Caching layer of backend |
## Product Configuration
| Variable | Mandatory | Description | If Not Set |
| -------- | --------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
| `DB_QUERY_LIMIT_DEFAULT` | No | Default pagination limit for data tables. | Defaults to `25`. Maximum is `100` |
| `DB_QUERY_LIMIT_GROUP_BY_GROUP` | No | Number of groups per page. | Defaults to `10`. |
| `DB_QUERY_LIMIT_GROUP_BY_RECORD` | No | Number of records per group. | Defaults to `10`. |
| `DB_QUERY_LIMIT_MAX` | No | Maximum allowable pagination limit. | Defaults to `1000`. |
| `DB_QUERY_LIMIT_MIN` | No | Minimum allowable pagination limit. | Defaults to `10` |
| `NC_CONNECT_TO_EXTERNAL_DB_DISABLED` | No | Disables the ability to create bases on external databases. | |
| `NC_INVITE_ONLY_SIGNUP` | No | Disables public signup; signup is possible only via invitations. Integrated into the [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup) as of version 0.99.0. | |
| `NC_REQUEST_BODY_SIZE` | No | Maximum bytes allowed in the request body, based on [ExpressJS limits](https://expressjs.com/en/resources/middleware/body-parser.html#limit). | Defaults to `1048576` (1 MB). |
| `NC_EXPORT_MAX_TIMEOUT` | No | Sets a timeout in milliseconds for downloading CSVs in batches if not completed within this period. | Defaults to `5000` (5 seconds). |
| `NC_ALLOW_LOCAL_HOOKS` | No | Allows webhooks to call local network links, posing potential security risks. Set to `true` to enable; all other values are considered `false`. | Defaults to `false`. |
| `NC_SANITIZE_COLUMN_NAME` | No | Enables sanitization of column names during their creation to prevent SQL injection and other security issues. | Defaults to `true`. |
| `NC_TOOL_DIR` | No | Specifies the directory to store metadata and app-related files. In Docker setups, this maps to `/usr/app/data/` for mounting volumes. | Defaults to the current working directory. |
| `NC_MINIMAL_DBS` | No | Creates a new SQLite file for each base. All SQLite database files are stored in the `nc_minimal_dbs` folder. Enabling this also disables base creation on external databases. | |
| `NC_MIGRATIONS_DISABLED` | No | Disables NocoDB migrations. | |
| `NC_DISABLE_AUDIT` | No | Disables the audit log feature. | Defaults to `false`. |
| `NC_AUTOMATION_LOG_LEVEL` | No | Configures logging levels for automation features. Possible values: `OFF`, `ERROR`, `ALL`. More details can be found under [Webhooks](/automation/webhook/create-webhook). | Defaults to `OFF`. |
| Variable | Mandatory | Description | If Not Set |
| -------- | --------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
| `DB_QUERY_LIMIT_DEFAULT` | No | Default pagination limit for data tables. | Defaults to `25`. Maximum is `100` |
| `DB_QUERY_LIMIT_GROUP_BY_GROUP` | No | Number of groups per page. | Defaults to `10`. |
| `DB_QUERY_LIMIT_GROUP_BY_RECORD` | No | Number of records per group. | Defaults to `10`. |
| `DB_QUERY_LIMIT_MAX` | No | Maximum allowable pagination limit. | Defaults to `1000`. |
| `DB_QUERY_LIMIT_MIN` | No | Minimum allowable pagination limit. | Defaults to `10` |
| `NC_CONNECT_TO_EXTERNAL_DB_DISABLED` | No | Disables the ability to create bases on external databases. | |
| `NC_INVITE_ONLY_SIGNUP` | No | Disables public signup; signup is possible only via invitations. Integrated into the [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup) as of version 0.99.0. | |
| `NC_REQUEST_BODY_SIZE` | No | Maximum bytes allowed in the request body, based on [ExpressJS limits](https://expressjs.com/en/resources/middleware/body-parser.html#limit). | Defaults to `1048576` (1 MB). |
| `NC_EXPORT_MAX_TIMEOUT` | No | Sets a timeout in milliseconds for downloading CSVs in batches if not completed within this period. | Defaults to `5000` (5 seconds). |
| `NC_ALLOW_LOCAL_HOOKS` | No | Allows webhooks to call local network links, posing potential security risks. Set to `true` to enable; all other values are considered `false`. | Defaults to `false`. |
| `NC_SANITIZE_COLUMN_NAME` | No | Enables sanitization of column names during their creation to prevent SQL injection and other security issues. | Defaults to `true`. |
| `NC_TOOL_DIR` | No | Specifies the directory to store metadata and app-related files. In Docker setups, this maps to `/usr/app/data/` for mounting volumes. | Defaults to the current working directory. |
| `NC_MINIMAL_DBS` | No | Enables the minimal database feature of NocoDB. For more details, see [Minimal Database behavior](#minimal-database). | Enabled by default for PostgreSQL when the database user has schema creation permission. Set to `false` to disable. |
| `NC_MIGRATIONS_DISABLED` | No | Disables NocoDB migrations. | |
| `NC_DISABLE_AUDIT` | No | Disables the audit log feature. | Defaults to `false`. |
| `NC_AUTOMATION_LOG_LEVEL` | No | Configures logging levels for automation features. Possible values: `OFF`, `ERROR`, `ALL`. More details can be found under [Webhooks](/automation/webhook/create-webhook). | Defaults to `OFF`. |
### Minimal Database
This feature organizes base data into smaller, modular structures for PostgreSQL and SQLite databases:
- **SQLite**: Each base's data is stored as a separate SQLite file, ensuring isolated storage.
- **PostgreSQL**: A unique schema is created for each base, providing logical separation within the database.
**Note**: For PostgreSQL, this feature is enabled by default if the user has the required permissions. To disable it, set the `NC_MINIMAL_DBS` environment variable to `false`.
## Logging & Monitoring
| Variable | Mandatory | Description | If Not Set |

2
packages/nocodb/src/controllers/filters.controller.ts
Unescape View File

@ -38,7 +38,7 @@ export class FiltersController {
return new PagedResponseImpl(
await this.filtersService.filterList(context, {
viewId,
includeAllFilters: includeAllFilters === 'true'
includeAllFilters: includeAllFilters === 'true',
}),
);
}

8
packages/nocodb/src/db/BaseModelSqlv2.ts
Unescape View File

@ -210,6 +210,7 @@ class BaseModelSqlv2 {
protected source: Source;
public model: Model;
public context: NcContext;
public schema?: string;
public static config: any = defaultLimitConfig;
@ -222,14 +223,17 @@ class BaseModelSqlv2 {
model,
viewId,
context,
schema,
}: {
[key: string]: any;
model: Model;
schema?: string;
}) {
this._dbDriver = dbDriver;
this.model = model;
this.viewId = viewId;
this.context = context;
this.schema = schema;
autoBind(this);
}
@ -4854,7 +4858,9 @@ class BaseModelSqlv2 {
public getTnPath(tb: { table_name: string } | string, alias?: string) {
const tn = typeof tb === 'string' ? tb : tb.table_name;
const schema = (this.dbDriver as any).searchPath?.();
if (this.isMssql && schema) {
if (this.isPg && this.schema) {
return `${this.schema}.${tn}${alias ? ` as ${alias}` : ``}`;
} else if (this.isMssql && schema) {
return this.dbDriver.raw(`??.??${alias ? ' as ??' : ''}`, [
schema,
tn,

330
packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts
Unescape View File

@ -493,7 +493,7 @@ class PGClient extends KnexClient {
]);
}
const schemaName = this.connectionConfig.searchPath?.[0] || 'public';
const schemaName = this.getEffectiveSchema(args);
// Check schemaExists because `CREATE SCHEMA IF NOT EXISTS` requires permissions of `CREATE ON DATABASE`
const schemaExists = !!(
@ -524,6 +524,10 @@ class PGClient extends KnexClient {
return result;
}
protected getEffectiveSchema(args: { schema?: string } = {}) {
return args?.schema || this.schema;
}
async dropDatabase(args) {
const _func = this.dropDatabase.name;
const result = new Result();
@ -575,7 +579,11 @@ class PGClient extends KnexClient {
const exists = await this.sqlClient.raw(
`SELECT table_schema,table_name as tn, table_catalog FROM information_schema.tables where table_schema=? and
table_name = ? and table_catalog = ?`,
[this.schema, args.tn, this.connectionConfig.connection.database],
[
this.getEffectiveSchema(args),
args.tn,
this.connectionConfig.connection.database,
],
);
if (exists.rows.length === 0) {
@ -638,7 +646,11 @@ class PGClient extends KnexClient {
try {
const { rows } = await this.sqlClient.raw(
`SELECT table_schema,table_name as tn, table_catalog FROM information_schema.tables where table_schema=? and table_name = ? and table_catalog = ?'`,
[this.schema, args.tn, this.connectionConfig.connection.database],
[
this.getEffectiveSchema(args),
args.tn,
this.connectionConfig.connection.database,
],
);
result.data.value = rows.length > 0;
} catch (e) {
@ -716,7 +728,7 @@ class PGClient extends KnexClient {
FROM information_schema.tables
where table_schema = ?
ORDER BY table_schema, table_name`,
[this.schema],
[this.getEffectiveSchema(args)],
);
result.data.list = rows.filter(
@ -852,7 +864,7 @@ class PGClient extends KnexClient {
where c.table_catalog=:database and c.table_schema=:schema and c.table_name=:table
order by c.table_name, c.ordinal_position`,
{
schema: this.schema,
schema: this.getEffectiveSchema(args),
database: args.databaseName,
table: args.tn,
},
@ -933,7 +945,6 @@ class PGClient extends KnexClient {
return result;
}
/**
*
* @param {Object} - args - Input arguments
@ -994,7 +1005,7 @@ class PGClient extends KnexClient {
and i.oid<>0
AND f.attnum > 0
ORDER BY i.relname, f.attnum;`,
[this.schema, args.tn],
[this.getEffectiveSchema(args), args.tn],
);
result.data.list = rows;
} catch (e) {
@ -1024,9 +1035,13 @@ class PGClient extends KnexClient {
const foreignKeyName = args.foreignKeyName || null;
args.childTableWithSchema = args.childTable;
args.childTableWithSchema = args.schema
? `${args.schema}.${args.childTable}`
: args.childTable;
args.parentTableWithSchema = args.parentTable;
args.parentTableWithSchema = args.schema
? `${args.schema}.${args.parentTable}`
: args.parentTable;
try {
// const self = this;
@ -1185,7 +1200,7 @@ class PGClient extends KnexClient {
on pc.conname = tc.constraint_name
WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema=:schema and tc.table_name=:table
order by tc.table_name;`,
{ schema: this.schema, table: args.tn },
{ schema: this.getEffectiveSchema(args), table: args.tn },
);
const ruleMapping = {
@ -1212,6 +1227,74 @@ class PGClient extends KnexClient {
return result;
}
/**
*
* @param {Object} - args
* @param {String} - args.parentTable
* @param {String} - args.parentColumn
* @param {String} - args.childColumn
* @param {String} - args.childTable
* @returns {Promise<{upStatement, downStatement}>}
*/
async relationCreate(args) {
const _func = this.relationCreate.name;
const result = new Result();
log.api(`${_func}:args:`, args);
const foreignKeyName = args.foreignKeyName || null;
args.childTableWithSchema = args.schema
? `${args.schema}.${args.childTable}`
: args.childTable;
args.parentTableWithSchema = args.schema
? `${args.schema}.${args.parentTable}`
: args.parentTable;
try {
const upQb = this.sqlClient.schema.table(
args.childTableWithSchema,
function (table) {
table = table
.foreign(args.childColumn, foreignKeyName)
.references(args.parentColumn)
.on(args.parentTableWithSchema);
if (args.onUpdate) {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table.onDelete(args.onDelete);
}
},
);
await this.sqlClient.raw(upQb.toQuery());
const upStatement = this.querySeparator() + upQb.toQuery();
this.emit(`Success : ${upStatement}`);
const downStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.childTableWithSchema, function (table) {
table.dropForeign(args.childColumn, foreignKeyName);
})
.toQuery();
result.data.object = {
upStatement: [{ sql: upStatement }],
downStatement: [{ sql: downStatement }],
};
} catch (e) {
log.ppe(e, _func);
throw e;
}
return result;
}
/**
*
* @param {Object} - args - Input arguments
@ -1256,7 +1339,7 @@ class PGClient extends KnexClient {
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = ?
order by tc.table_name;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
const ruleMapping = {
@ -1309,7 +1392,7 @@ class PGClient extends KnexClient {
const { rows } = await this.sqlClient.raw(
`select * from information_schema.triggers where trigger_schema=? and event_object_table=?`,
[this.schema, args.tn],
[this.getEffectiveSchema(args), args.tn],
);
for (let i = 0; i < rows.length; ++i) {
@ -1353,13 +1436,13 @@ class PGClient extends KnexClient {
try {
args.databaseName = this.connectionConfig.connection.database;
const { rows } = await this.raw(
const { rows } = await this.sqlClient.raw(
`SELECT *
FROM pg_catalog.pg_namespace n
JOIN pg_catalog.pg_proc p
ON pronamespace = n.oid
WHERE nspname = ?;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
const functionRows = [];
for (let i = 0; i < rows.length; ++i) {
@ -1408,13 +1491,13 @@ class PGClient extends KnexClient {
try {
args.databaseName = this.connectionConfig.connection.database;
const { rows } = await this.raw(
const { rows } = await this.sqlClient.raw(
`SELECT *
FROM pg_catalog.pg_namespace n
JOIN pg_catalog.pg_proc p
ON pronamespace = n.oid
WHERE nspname = ?;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
const procedureRows = [];
for (let i = 0; i < rows.length; ++i) {
@ -1456,7 +1539,7 @@ class PGClient extends KnexClient {
`select *
from INFORMATION_SCHEMA.views
WHERE table_schema = ?;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
for (let i = 0; i < rows.length; ++i) {
@ -1494,7 +1577,7 @@ class PGClient extends KnexClient {
`SELECT format('%I.%I(%s)', ns.nspname, p.proname, oidvectortypes(p.proargtypes)) as function_declaration, pg_get_functiondef(p.oid) as create_function
FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = ? and p.proname = ?;`,
[this.schema, args.function_name],
[this.getEffectiveSchema(args), args.function_name],
);
// log.debug(response);
@ -2229,8 +2312,12 @@ class PGClient extends KnexClient {
for (let i = 0; i < args.columns.length; i++) {
const column = args.columns[i];
if (column.au) {
const triggerFnName = `xc_au_${args.tn}_${column.cn}`;
const triggerName = `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnName = args.schema
? `xc_au_${args.schema}_${args.tn}_${column.cn}`
: `xc_au_${args.tn}_${column.cn}`;
const triggerName = args.schema
? `xc_trigger_${args.schema}_${args.tn}_${column.cn}`
: `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnQuery = this.genQuery(
`CREATE OR REPLACE FUNCTION ??()
@ -2252,14 +2339,18 @@ class PGClient extends KnexClient {
BEFORE UPDATE ON ??
FOR EACH ROW
EXECUTE PROCEDURE ??();`,
[triggerName, args.tn, triggerFnName],
[
triggerName,
args.schema ? `${args.schema}.${args.tn}` : args.tn,
triggerFnName,
],
);
downQuery +=
this.querySeparator() +
this.genQuery(`DROP TRIGGER IF EXISTS ?? ON ??;`, [
triggerName,
args.tn,
args.schema ? `${args.schema}.${args.tn}` : args.tn,
]) +
this.querySeparator() +
this.genQuery(`DROP FUNCTION IF EXISTS ??()`, [triggerFnName]);
@ -2280,8 +2371,12 @@ class PGClient extends KnexClient {
for (let i = 0; i < args.columns.length; i++) {
const column = args.columns[i];
if (column.au && column.altered === 1) {
const triggerFnName = `xc_au_${args.tn}_${column.cn}`;
const triggerName = `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnName = args.schema
? `xc_au_${args.schema}_${args.tn}_${column.cn}`
: `xc_au_${args.tn}_${column.cn}`;
const triggerName = args.schema
? `xc_trigger_${args.schema}_${args.tn}_${column.cn}`
: `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnQuery = this.genQuery(
`CREATE OR REPLACE FUNCTION ??()
@ -2303,7 +2398,11 @@ class PGClient extends KnexClient {
BEFORE UPDATE ON ??
FOR EACH ROW
EXECUTE PROCEDURE ??();`,
[triggerName, args.tn, triggerFnName],
[
triggerName,
args.schema ? `${args.schema}.${args.tn}` : args.tn,
triggerFnName,
],
);
downQuery +=
@ -2356,7 +2455,7 @@ class PGClient extends KnexClient {
log.api(`${_func}:args:`, args);
try {
args.table = args.tn;
args.table = args.schema ? `${args.schema}.${args.tn}` : args.tn;
const originalColumns = args.originalColumns;
args.connectionConfig = this._connectionConfig;
args.sqlClient = this.sqlClient;
@ -2477,7 +2576,9 @@ class PGClient extends KnexClient {
/** ************** create up & down statements *************** */
const upStatement =
this.querySeparator() +
this.sqlClient.schema.dropTable(args.tn).toString();
this.sqlClient.schema
.dropTable(args.schema ? `${args.schema}.${args.tn}` : args.tn)
.toString();
let downQuery = this.createTable(args.tn, args);
/**
@ -2563,7 +2664,9 @@ class PGClient extends KnexClient {
/** ************** drop tn *************** */
await this.sqlClient.raw(
this.sqlClient.schema.dropTable(args.tn).toQuery(),
this.sqlClient.schema
.dropTable(args.schema ? `${args.schema}.${args.tn}` : args.tn)
.toQuery(),
);
/** ************** return files *************** */
@ -2838,7 +2941,9 @@ class PGClient extends KnexClient {
query += this.alterTablePK(table, args.columns, [], query, true);
query = this.genQuery(`CREATE TABLE ?? (${query});`, [args.tn]);
query = this.genQuery(`CREATE TABLE ?? (${query});`, [
args.schema ? `${args.schema}.${args.tn}` : args.tn,
]);
return query;
}
@ -2954,12 +3059,7 @@ class PGClient extends KnexClient {
}
get schema() {
return (
(this.connectionConfig &&
this.connectionConfig.searchPath &&
this.connectionConfig.searchPath[0]) ||
'public'
);
return this.connectionConfig?.searchPath?.[0] || 'public';
}
/**
@ -3025,7 +3125,10 @@ class PGClient extends KnexClient {
await this.sqlClient.raw(
this.sqlClient.schema
.renameTable(
this.sqlClient.raw('??.??', [this.schema, args.tn_old]),
this.sqlClient.raw('??.??', [
this.getEffectiveSchema(args),
args.tn_old,
]),
args.tn,
)
.toQuery(),
@ -3036,7 +3139,10 @@ class PGClient extends KnexClient {
this.querySeparator() +
this.sqlClient.schema
.renameTable(
this.sqlClient.raw('??.??', [this.schema, args.tn]),
this.sqlClient.raw('??.??', [
this.getEffectiveSchema(args),
args.tn,
]),
args.tn_old,
)
.toQuery();
@ -3047,7 +3153,10 @@ class PGClient extends KnexClient {
this.querySeparator() +
this.sqlClient.schema
.renameTable(
this.sqlClient.raw('??.??', [this.schema, args.tn_old]),
this.sqlClient.raw('??.??', [
this.getEffectiveSchema(args),
args.tn_old,
]),
args.tn,
)
.toQuery();
@ -3064,6 +3173,151 @@ class PGClient extends KnexClient {
return result;
}
/**
*
* @param {Object} - args
* @param {String} - args.tn
* @param {String} - args.indexName
* @param {String} - args.non_unique
* @param {String[]} - args.columns
* @returns {Promise<{upStatement, downStatement}>}
*/
async indexCreate(args) {
const _func = this.indexCreate.name;
const result = new Result();
log.api(`${_func}:args:`, args);
const indexName = args.indexName || null;
try {
args.table = args.schema ? `${args.schema}.${args.tn}` : args.tn;
// s = await this.sqlClient.schema.index(Object.keys(args.columns));
await this.sqlClient.raw(
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique) {
table.index(args.columns, indexName);
} else {
table.unique(args.columns, indexName);
}
})
.toQuery(),
);
const upStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique) {
table.index(args.columns, indexName);
} else {
table.unique(args.columns, indexName);
}
})
.toQuery();
this.emit(`Success : ${upStatement}`);
const downStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique) {
table.dropIndex(args.columns, indexName);
} else {
table.dropUnique(args.columns, indexName);
}
})
.toQuery();
result.data.object = {
upStatement: [{ sql: upStatement }],
downStatement: [{ sql: downStatement }],
};
// result.data.object = {
// upStatement,
// downStatement
// };
} catch (e) {
log.ppe(e, _func);
throw e;
}
return result;
}
/**
*
* @param {Object} - args
* @param {String} - args.tn
* @param {String[]} - args.columns
* @param {String} - args.indexName
* @param {String} - args.non_unique
* @returns {Promise<{upStatement, downStatement}>}
*/
async indexDelete(args) {
const _func = this.indexDelete.name;
const result = new Result();
log.api(`${_func}:args:`, args);
const indexName = args.indexName || null;
try {
args.table = args.schema ? `${args.schema}.${args.tn}` : args.tn;
// s = await this.sqlClient.schema.index(Object.keys(args.columns));
await this.sqlClient.raw(
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique_original) {
table.dropIndex(args.columns, indexName);
} else {
table.dropUnique(args.columns, indexName);
}
})
.toQuery(),
);
const upStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique_original) {
table.dropIndex(args.columns, indexName);
} else {
table.dropUnique(args.columns, indexName);
}
})
.toQuery();
this.emit(`Success : ${upStatement}`);
const downStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique_original) {
table.index(args.columns, indexName);
} else {
table.unique(args.columns, indexName);
}
})
.toQuery();
result.data.object = {
upStatement: [{ sql: upStatement }],
downStatement: [{ sql: downStatement }],
};
} catch (e) {
log.ppe(e, _func);
throw e;
}
return result;
}
}
export default PGClient;

57
packages/nocodb/src/helpers/initBaseBehaviour.ts
Unescape View File

@ -0,0 +1,57 @@
import { Logger } from '@nestjs/common';
import { Knex } from 'knex';
import PgConnectionConfig = Knex.PgConnectionConfig;
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import CustomKnex from '~/db/CustomKnex';
const logger = new Logger('initBaseBehavior');
export async function initBaseBehavior() {
const dataConfig = await NcConnectionMgrv2.getDataConfig();
// return if client is not postgres
if (dataConfig.client !== 'pg') {
return;
}
// if NC_MINIMAL_DBS already exists, return
if (process.env.NC_MINIMAL_DBS === 'false') {
return;
}
let tempConnection: Knex<any, unknown[]> | undefined;
try {
tempConnection = CustomKnex(dataConfig);
// check if database user have permission to create new schema
const schemaCreateAllowed = await tempConnection.raw(
"SELECT has_database_privilege(:user, :database, 'CREATE') as has_database_privilege",
{
database: (dataConfig.connection as PgConnectionConfig).database,
user: (dataConfig.connection as PgConnectionConfig).user,
},
);
// if schema creation is not allowed, return
if (!schemaCreateAllowed.rows[0]?.has_database_privilege) {
// set NC_MINIMAL_DBS to false if it's set to true and log warning
if (process.env.NC_MINIMAL_DBS === 'true') {
process.env.NC_MINIMAL_DBS = 'false';
}
logger.warn(
`User ${(dataConfig.connection as PgConnectionConfig)?.user} does not have permission to create schema, minimal databases feature will be disabled`,
);
return;
}
// set NC_MINIMAL_DBS to true
process.env.NC_MINIMAL_DBS = 'true';
} catch (error) {
logger.warn(
`Error while checking schema creation permission: ${error.message}`,
);
} finally {
// close the connection since it's only used to verify permission
await tempConnection?.destroy();
}
}

4
packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
Unescape View File

@ -50,6 +50,7 @@ import * as nc_060_descriptions from '~/meta/migrations/v2/nc_060_descriptions';
import * as nc_061_integration_is_default from '~/meta/migrations/v2/nc_061_integration_is_default';
import * as nc_062_integration_store from '~/meta/migrations/v2/nc_062_integration_store';
import * as nc_063_form_field_filter from '~/meta/migrations/v2/nc_063_form_field_filter';
import * as nc_064_pg_minimal_dbs from '~/meta/migrations/v2/nc_064_pg_minimal_dbs';
// Create a custom migration source class
export default class XcMigrationSourcev2 {
@ -111,6 +112,7 @@ export default class XcMigrationSourcev2 {
'nc_061_integration_is_default',
'nc_062_integration_store',
'nc_063_form_field_filter',
'nc_064_pg_minimal_dbs',
]);
}
@ -224,6 +226,8 @@ export default class XcMigrationSourcev2 {
return nc_062_integration_store;
case 'nc_063_form_field_filter':
return nc_063_form_field_filter;
case 'nc_064_pg_minimal_dbs':
return nc_064_pg_minimal_dbs;
}
}
}

16
packages/nocodb/src/meta/migrations/v2/nc_064_pg_minimal_dbs.ts
Unescape View File

@ -0,0 +1,16 @@
import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => {
table.boolean('is_local').defaultTo(false);
});
};
const down = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => {
table.dropColumn('is_local');
});
};
export { up, down };

13
packages/nocodb/src/models/Model.ts
Unescape View File

@ -34,6 +34,7 @@ import {
prepareForDb,
prepareForResponse,
} from '~/utils/modelUtils';
import { Source } from '~/models';
const logger = new Logger('Model');
@ -471,21 +472,33 @@ export default class Model implements TableType {
dbDriver: XKnex;
model?: Model;
extractDefaultView?: boolean;
source?: Source;
},
ncMeta = Noco.ncMeta,
): Promise<BaseModelSqlv2> {
const model = args?.model || (await this.get(context, args.id, ncMeta));
const source =
args.source ||
(await Source.get(context, model.source_id, false, ncMeta));
if (!args?.viewId && args.extractDefaultView) {
const view = await View.getDefaultView(context, model.id, ncMeta);
args.viewId = view.id;
}
let schema: string;
if (source?.isMeta(true, 1)) {
schema = source.getConfig()?.schema;
} else if (source?.type === 'pg') {
schema = source.getConfig()?.searchPath?.[0];
}
return new BaseModelSqlv2({
context,
dbDriver: args.dbDriver,
viewId: args.viewId,
model,
schema,
});
}

17
packages/nocodb/src/models/Source.ts
Unescape View File

@ -34,6 +34,7 @@ export default class Source implements SourceType {
alias?: string;
type?: DriverClient;
is_meta?: BoolType;
is_local?: BoolType;
is_schema_readonly?: BoolType;
is_data_readonly?: BoolType;
config?: string;
@ -71,6 +72,7 @@ export default class Source implements SourceType {
'config',
'type',
'is_meta',
'is_local',
'inflection_column',
'inflection_table',
'order',
@ -131,6 +133,7 @@ export default class Source implements SourceType {
'config',
'type',
'is_meta',
'is_local',
'inflection_column',
'inflection_table',
'order',
@ -297,6 +300,15 @@ export default class Source implements SourceType {
}
public async getConnectionConfig(): Promise<any> {
if (this.is_meta || this.is_local) {
const metaConfig = await NcConnectionMgrv2.getDataConfig();
const config = { ...metaConfig };
if (config.client === 'sqlite3') {
config.connection = metaConfig;
}
return config;
}
const config = this.getConfig();
// todo: update sql-client args
@ -307,7 +319,6 @@ export default class Source implements SourceType {
return config;
}
public getConfig(skipIntegrationConfig = false): any {
if (this.is_meta) {
const metaConfig = Noco.getConfig()?.meta?.db;
@ -556,9 +567,9 @@ export default class Source implements SourceType {
if (_mode === 0) {
return this.is_meta;
}
return false;
return this.is_local;
} else {
return this.is_meta;
return this.is_meta || this.is_local;
}
}

7
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
Unescape View File

@ -466,12 +466,7 @@ export class AtImportProcessor {
(value as any).name = 'nc_empty';
}
// skip duplicates (we don't allow them)
if (
options.find(
(el) =>
el.title === (value as any).name,
)
) {
if (options.find((el) => el.title === (value as any).name)) {
logWarning(
`Duplicate select option found: ${col.name} :: ${
(value as any).name

1
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
Unescape View File

@ -243,7 +243,6 @@ async function readView(viewId) {
resolve(fullObject);
});
});
if (data?.data) {
return { view: data.data };

4
packages/nocodb/src/providers/init-meta-service.provider.ts
Unescape View File

@ -13,6 +13,7 @@ import { User } from '~/models';
import { NcConfig, prepareEnv } from '~/utils/nc-config';
import { MetaTable, RootScopes } from '~/utils/globals';
import { updateMigrationJobsState } from '~/helpers/migrationJobs';
import { initBaseBehavior } from '~/helpers/initBaseBehaviour';
export const InitMetaServiceProvider: FactoryProvider = {
// initialize app,
@ -83,6 +84,9 @@ export const InitMetaServiceProvider: FactoryProvider = {
Noco.config = config;
Noco.eventEmitter = eventEmitter;
// decide base behavior based on env and database permissions
await initBaseBehavior();
if (!instanceConfig) {
// bump to latest version for fresh install
await updateMigrationJobsState({

72
packages/nocodb/src/services/bases.service.ts
Unescape View File

@ -163,38 +163,56 @@ export class BasesService {
baseBody.prefix = `nc_${ranId}__`;
baseBody.is_meta = true;
if (process.env.NC_MINIMAL_DBS === 'true') {
// if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each base
// each file will be named as nc_<random_id>.db
const fs = require('fs');
const toolDir = getToolDir();
const nanoidv2 = customAlphabet(
'1234567890abcdefghijklmnopqrstuvwxyz',
14,
);
if (!(await promisify(fs.exists)(`${toolDir}/nc_minimal_dbs`))) {
await promisify(fs.mkdir)(`${toolDir}/nc_minimal_dbs`);
}
const dbId = nanoidv2();
const baseTitle = DOMPurify.sanitize(baseBody.title);
baseBody.prefix = '';
baseBody.sources = [
{
type: 'sqlite3',
is_meta: false,
config: {
client: 'sqlite3',
connection: {
const dataConfig = await Noco.getConfig()?.meta?.db;
if (dataConfig?.client === 'pg') {
baseBody.prefix = '';
baseBody.sources = [
{
type: 'pg',
is_local: true,
is_meta: false,
config: {
schema: baseId,
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
];
} else {
// if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each base
// each file will be named as nc_<random_id>.db
const fs = require('fs');
const toolDir = getToolDir();
const nanoidv2 = customAlphabet(
'1234567890abcdefghijklmnopqrstuvwxyz',
14,
);
if (!(await promisify(fs.exists)(`${toolDir}/nc_minimal_dbs`))) {
await promisify(fs.mkdir)(`${toolDir}/nc_minimal_dbs`);
}
const dbId = nanoidv2();
const baseTitle = DOMPurify.sanitize(baseBody.title);
baseBody.prefix = '';
baseBody.sources = [
{
type: 'sqlite3',
is_meta: false,
is_local: true,
config: {
client: 'sqlite3',
database: baseTitle,
connection: {
filename: `${toolDir}/nc_minimal_dbs/${baseTitle}_${dbId}.db`,
client: 'sqlite3',
database: baseTitle,
connection: {
filename: `${toolDir}/nc_minimal_dbs/${baseTitle}_${dbId}.db`,
},
},
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
];
];
}
} else {
const db = Noco.getConfig().meta?.db;
baseBody.sources = [

7
packages/nocodb/src/services/data-alias-nested.service.ts
Unescape View File

@ -32,6 +32,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -80,6 +81,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -123,6 +125,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -166,6 +169,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -208,6 +212,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -253,6 +258,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -300,6 +306,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);

2
packages/nocodb/src/services/data-table.service.ts
Unescape View File

@ -52,6 +52,7 @@ export class DataTableService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const row = await baseModel.readByPk(param.rowId, false, param.query, {
@ -82,6 +83,7 @@ export class DataTableService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
if (view.type !== ViewTypes.GRID) {

23
packages/nocodb/src/services/datas.service.ts
Unescape View File

@ -96,6 +96,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const countArgs: any = { ...param.query, throwErrorIfInvalidParams: true };
@ -124,6 +125,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.nestedInsert(param.body, null, param.cookie);
@ -145,6 +147,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.updateByPk(
@ -166,6 +169,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
// if xcdb base skip checking for LTAR
@ -209,6 +213,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
}));
const { ast, dependencyFields } = await getAst(context, {
@ -273,6 +278,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const args: any = { ...query };
@ -305,6 +311,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...query };
@ -342,6 +349,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const row = await baseModel.readByPk(param.rowId, false, param.query, {
getHiddenColumn: param.getHiddenColumn,
@ -366,6 +374,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.exist(param.rowId);
@ -403,6 +412,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -487,6 +497,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `${model.title}List`;
@ -551,6 +562,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = 'List';
@ -615,6 +627,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = 'List';
@ -679,6 +692,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = 'List';
@ -743,6 +757,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `${model.title}List`;
@ -797,6 +812,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -830,6 +846,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.insert(param.body, null, param.cookie);
@ -854,6 +871,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.updateByPk(
@ -882,6 +900,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.delByPk(param.rowId, null, param.cookie);
@ -911,6 +930,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
await baseModel.removeChild({
@ -947,6 +967,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
await baseModel.addChild({
@ -1010,6 +1031,7 @@ export class DatasService {
id: view.model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { offset, dbRows, elapsed } = await getDbRows(context, {
@ -1048,6 +1070,7 @@ export class DatasService {
id: view.model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { offset, dbRows, elapsed } = await getDbRows(context, {

10
packages/nocodb/src/services/public-datas.service.ts
Unescape View File

@ -65,6 +65,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -129,6 +130,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...param.query };
@ -198,6 +200,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast } = await getAst(context, { model, query: param.query, view });
@ -293,6 +296,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...query };
@ -350,6 +354,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
await view.getViewWithInfo(context);
@ -470,6 +475,7 @@ export class PublicDatasService {
id: model.id,
viewId: colOptions.fk_target_view_id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -558,6 +564,7 @@ export class PublicDatasService {
id: view.fk_model_id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `List`;
@ -637,6 +644,7 @@ export class PublicDatasService {
id: view.fk_model_id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `List`;
@ -711,6 +719,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const row = await baseModel.readByPk(rowId, false, query);
@ -813,6 +822,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...param.query };

4
packages/nocodb/src/utils/common/NcConnectionMgrv2.ts
Unescape View File

@ -111,4 +111,8 @@ export default class NcConnectionMgrv2 {
...(await source.getConnectionConfig()),
});
}
public static async getDataConfig?() {
return Noco.getConfig()?.meta?.db;
}
}

Write Preview
Loading…
Cancel
Save