Browse Source

Merge branch 'develop' into nc-fix/swagger-base-pw

pull/9543/head
Raju Udava 2 months ago
parent
commit
d1ccb70bc4
  1. 27
      docker-compose/1_Auto_Upstall/noco.sh
  2. 14
      packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md
  3. 2
      packages/nocodb/src/controllers/filters.controller.ts
  4. 8
      packages/nocodb/src/db/BaseModelSqlv2.ts
  5. 330
      packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts
  6. 57
      packages/nocodb/src/helpers/initBaseBehaviour.ts
  7. 4
      packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
  8. 16
      packages/nocodb/src/meta/migrations/v2/nc_064_pg_minimal_dbs.ts
  9. 13
      packages/nocodb/src/models/Model.ts
  10. 17
      packages/nocodb/src/models/Source.ts
  11. 7
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  12. 1
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
  13. 4
      packages/nocodb/src/providers/init-meta-service.provider.ts
  14. 18
      packages/nocodb/src/services/bases.service.ts
  15. 7
      packages/nocodb/src/services/data-alias-nested.service.ts
  16. 2
      packages/nocodb/src/services/data-table.service.ts
  17. 23
      packages/nocodb/src/services/datas.service.ts
  18. 10
      packages/nocodb/src/services/public-datas.service.ts
  19. 4
      packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

27
docker-compose/1_Auto_Upstall/noco.sh

@ -496,10 +496,19 @@ generate_credentials() {
create_docker_compose_file() {
if [ "${CONFIG_EDITION}" = "EE" ] || [ "${CONFIG_EDITION}" = "ee" ]; then
image="nocodb/nocodb-ee:latest"
else
image="nocodb/nocodb:latest"
fi
if [ "${CONFIG_EDITION}" = "EE" ] || [ "${CONFIG_EDITION}" = "ee" ]; then
image="nocodb/ee:latest"
# for easier string interpolation
if [ "${CONFIG_REDIS_ENABLED}" = "Y" ]; then
gen_redis=1
fi
if [ "${CONFIG_MINIO_ENABLED}" = "Y" ]; then
gen_minio=1
fi
local compose_file="docker-compose.yml"
@ -514,8 +523,8 @@ services:
replicas: ${CONFIG_NUM_INSTANCES}
depends_on:
- db
${CONFIG_REDIS_ENABLED:+- redis}
${CONFIG_MINIO_ENABLED:+- minio}
${gen_redis:+- redis}
${gen_minio:+- minio}
restart: unless-stopped
volumes:
- ./nocodb:/usr/app/data
@ -636,7 +645,6 @@ EOF
- "traefik.http.routers.minio.rule=Host(\`${CONFIG_MINIO_DOMAIN_NAME}\`)"
EOF
# If minio SSL is enabled, set the entry point to websecure
fi
if [ "$CONFIG_MINIO_SSL_ENABLED" = "Y" ]; then
cat >> "$compose_file" <<EOF
- "traefik.http.routers.minio.entrypoints=websecure"
@ -660,6 +668,7 @@ EOF
- nocodb-network
EOF
fi
if [ "${CONFIG_WATCHTOWER_ENABLED}" = "Y" ]; then
cat >> "$compose_file" <<EOF
watchtower:
@ -674,10 +683,14 @@ EOF
EOF
fi
cat >> "$compose_file" <<EOF
if [ "$CONFIG_REDIS_ENABLED" = "Y" ]; then
cat >> "$compose_file" <<EOF
volumes:
${CONFIG_REDIS_ENABLED:+redis:}
redis:
EOF
fi
cat >> "$compose_file" <<EOF
networks:
nocodb-network:
driver: bridge

14
packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md

@ -76,7 +76,7 @@ For production use cases, it is crucial to set all environment variables marked
## Product Configuration
| Variable | Mandatory | Description | If Not Set |
| -------- | --------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
| -------- | --------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
| `DB_QUERY_LIMIT_DEFAULT` | No | Default pagination limit for data tables. | Defaults to `25`. Maximum is `100` |
| `DB_QUERY_LIMIT_GROUP_BY_GROUP` | No | Number of groups per page. | Defaults to `10`. |
| `DB_QUERY_LIMIT_GROUP_BY_RECORD` | No | Number of records per group. | Defaults to `10`. |
@ -89,11 +89,21 @@ For production use cases, it is crucial to set all environment variables marked
| `NC_ALLOW_LOCAL_HOOKS` | No | Allows webhooks to call local network links, posing potential security risks. Set to `true` to enable; all other values are considered `false`. | Defaults to `false`. |
| `NC_SANITIZE_COLUMN_NAME` | No | Enables sanitization of column names during their creation to prevent SQL injection and other security issues. | Defaults to `true`. |
| `NC_TOOL_DIR` | No | Specifies the directory to store metadata and app-related files. In Docker setups, this maps to `/usr/app/data/` for mounting volumes. | Defaults to the current working directory. |
| `NC_MINIMAL_DBS` | No | Creates a new SQLite file for each base. All SQLite database files are stored in the `nc_minimal_dbs` folder. Enabling this also disables base creation on external databases. | |
| `NC_MINIMAL_DBS` | No | Enables the minimal database feature of NocoDB. For more details, see [Minimal Database behavior](#minimal-database). | Enabled by default for PostgreSQL when the database user has schema creation permission. Set to `false` to disable. |
| `NC_MIGRATIONS_DISABLED` | No | Disables NocoDB migrations. | |
| `NC_DISABLE_AUDIT` | No | Disables the audit log feature. | Defaults to `false`. |
| `NC_AUTOMATION_LOG_LEVEL` | No | Configures logging levels for automation features. Possible values: `OFF`, `ERROR`, `ALL`. More details can be found under [Webhooks](/automation/webhook/create-webhook). | Defaults to `OFF`. |
### Minimal Database
This feature organizes base data into smaller, modular structures for PostgreSQL and SQLite databases:
- **SQLite**: Each base's data is stored as a separate SQLite file, ensuring isolated storage.
- **PostgreSQL**: A unique schema is created for each base, providing logical separation within the database.
**Note**: For PostgreSQL, this feature is enabled by default if the user has the required permissions. To disable it, set the `NC_MINIMAL_DBS` environment variable to `false`.
## Logging & Monitoring
| Variable | Mandatory | Description | If Not Set |
| -------- | --------- |---------------------------------------------------------------------------------------|------------|

2
packages/nocodb/src/controllers/filters.controller.ts

@ -38,7 +38,7 @@ export class FiltersController {
return new PagedResponseImpl(
await this.filtersService.filterList(context, {
viewId,
includeAllFilters: includeAllFilters === 'true'
includeAllFilters: includeAllFilters === 'true',
}),
);
}

8
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -210,6 +210,7 @@ class BaseModelSqlv2 {
protected source: Source;
public model: Model;
public context: NcContext;
public schema?: string;
public static config: any = defaultLimitConfig;
@ -222,14 +223,17 @@ class BaseModelSqlv2 {
model,
viewId,
context,
schema,
}: {
[key: string]: any;
model: Model;
schema?: string;
}) {
this._dbDriver = dbDriver;
this.model = model;
this.viewId = viewId;
this.context = context;
this.schema = schema;
autoBind(this);
}
@ -4854,7 +4858,9 @@ class BaseModelSqlv2 {
public getTnPath(tb: { table_name: string } | string, alias?: string) {
const tn = typeof tb === 'string' ? tb : tb.table_name;
const schema = (this.dbDriver as any).searchPath?.();
if (this.isMssql && schema) {
if (this.isPg && this.schema) {
return `${this.schema}.${tn}${alias ? ` as ${alias}` : ``}`;
} else if (this.isMssql && schema) {
return this.dbDriver.raw(`??.??${alias ? ' as ??' : ''}`, [
schema,
tn,

330
packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts

@ -493,7 +493,7 @@ class PGClient extends KnexClient {
]);
}
const schemaName = this.connectionConfig.searchPath?.[0] || 'public';
const schemaName = this.getEffectiveSchema(args);
// Check schemaExists because `CREATE SCHEMA IF NOT EXISTS` requires permissions of `CREATE ON DATABASE`
const schemaExists = !!(
@ -524,6 +524,10 @@ class PGClient extends KnexClient {
return result;
}
protected getEffectiveSchema(args: { schema?: string } = {}) {
return args?.schema || this.schema;
}
async dropDatabase(args) {
const _func = this.dropDatabase.name;
const result = new Result();
@ -575,7 +579,11 @@ class PGClient extends KnexClient {
const exists = await this.sqlClient.raw(
`SELECT table_schema,table_name as tn, table_catalog FROM information_schema.tables where table_schema=? and
table_name = ? and table_catalog = ?`,
[this.schema, args.tn, this.connectionConfig.connection.database],
[
this.getEffectiveSchema(args),
args.tn,
this.connectionConfig.connection.database,
],
);
if (exists.rows.length === 0) {
@ -638,7 +646,11 @@ class PGClient extends KnexClient {
try {
const { rows } = await this.sqlClient.raw(
`SELECT table_schema,table_name as tn, table_catalog FROM information_schema.tables where table_schema=? and table_name = ? and table_catalog = ?'`,
[this.schema, args.tn, this.connectionConfig.connection.database],
[
this.getEffectiveSchema(args),
args.tn,
this.connectionConfig.connection.database,
],
);
result.data.value = rows.length > 0;
} catch (e) {
@ -716,7 +728,7 @@ class PGClient extends KnexClient {
FROM information_schema.tables
where table_schema = ?
ORDER BY table_schema, table_name`,
[this.schema],
[this.getEffectiveSchema(args)],
);
result.data.list = rows.filter(
@ -852,7 +864,7 @@ class PGClient extends KnexClient {
where c.table_catalog=:database and c.table_schema=:schema and c.table_name=:table
order by c.table_name, c.ordinal_position`,
{
schema: this.schema,
schema: this.getEffectiveSchema(args),
database: args.databaseName,
table: args.tn,
},
@ -933,7 +945,6 @@ class PGClient extends KnexClient {
return result;
}
/**
*
* @param {Object} - args - Input arguments
@ -994,7 +1005,7 @@ class PGClient extends KnexClient {
and i.oid<>0
AND f.attnum > 0
ORDER BY i.relname, f.attnum;`,
[this.schema, args.tn],
[this.getEffectiveSchema(args), args.tn],
);
result.data.list = rows;
} catch (e) {
@ -1024,9 +1035,13 @@ class PGClient extends KnexClient {
const foreignKeyName = args.foreignKeyName || null;
args.childTableWithSchema = args.childTable;
args.childTableWithSchema = args.schema
? `${args.schema}.${args.childTable}`
: args.childTable;
args.parentTableWithSchema = args.parentTable;
args.parentTableWithSchema = args.schema
? `${args.schema}.${args.parentTable}`
: args.parentTable;
try {
// const self = this;
@ -1185,7 +1200,7 @@ class PGClient extends KnexClient {
on pc.conname = tc.constraint_name
WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema=:schema and tc.table_name=:table
order by tc.table_name;`,
{ schema: this.schema, table: args.tn },
{ schema: this.getEffectiveSchema(args), table: args.tn },
);
const ruleMapping = {
@ -1212,6 +1227,74 @@ class PGClient extends KnexClient {
return result;
}
/**
*
* @param {Object} - args
* @param {String} - args.parentTable
* @param {String} - args.parentColumn
* @param {String} - args.childColumn
* @param {String} - args.childTable
* @returns {Promise<{upStatement, downStatement}>}
*/
async relationCreate(args) {
const _func = this.relationCreate.name;
const result = new Result();
log.api(`${_func}:args:`, args);
const foreignKeyName = args.foreignKeyName || null;
args.childTableWithSchema = args.schema
? `${args.schema}.${args.childTable}`
: args.childTable;
args.parentTableWithSchema = args.schema
? `${args.schema}.${args.parentTable}`
: args.parentTable;
try {
const upQb = this.sqlClient.schema.table(
args.childTableWithSchema,
function (table) {
table = table
.foreign(args.childColumn, foreignKeyName)
.references(args.parentColumn)
.on(args.parentTableWithSchema);
if (args.onUpdate) {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table.onDelete(args.onDelete);
}
},
);
await this.sqlClient.raw(upQb.toQuery());
const upStatement = this.querySeparator() + upQb.toQuery();
this.emit(`Success : ${upStatement}`);
const downStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.childTableWithSchema, function (table) {
table.dropForeign(args.childColumn, foreignKeyName);
})
.toQuery();
result.data.object = {
upStatement: [{ sql: upStatement }],
downStatement: [{ sql: downStatement }],
};
} catch (e) {
log.ppe(e, _func);
throw e;
}
return result;
}
/**
*
* @param {Object} - args - Input arguments
@ -1256,7 +1339,7 @@ class PGClient extends KnexClient {
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = ?
order by tc.table_name;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
const ruleMapping = {
@ -1309,7 +1392,7 @@ class PGClient extends KnexClient {
const { rows } = await this.sqlClient.raw(
`select * from information_schema.triggers where trigger_schema=? and event_object_table=?`,
[this.schema, args.tn],
[this.getEffectiveSchema(args), args.tn],
);
for (let i = 0; i < rows.length; ++i) {
@ -1353,13 +1436,13 @@ class PGClient extends KnexClient {
try {
args.databaseName = this.connectionConfig.connection.database;
const { rows } = await this.raw(
const { rows } = await this.sqlClient.raw(
`SELECT *
FROM pg_catalog.pg_namespace n
JOIN pg_catalog.pg_proc p
ON pronamespace = n.oid
WHERE nspname = ?;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
const functionRows = [];
for (let i = 0; i < rows.length; ++i) {
@ -1408,13 +1491,13 @@ class PGClient extends KnexClient {
try {
args.databaseName = this.connectionConfig.connection.database;
const { rows } = await this.raw(
const { rows } = await this.sqlClient.raw(
`SELECT *
FROM pg_catalog.pg_namespace n
JOIN pg_catalog.pg_proc p
ON pronamespace = n.oid
WHERE nspname = ?;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
const procedureRows = [];
for (let i = 0; i < rows.length; ++i) {
@ -1456,7 +1539,7 @@ class PGClient extends KnexClient {
`select *
from INFORMATION_SCHEMA.views
WHERE table_schema = ?;`,
[this.schema],
[this.getEffectiveSchema(args)],
);
for (let i = 0; i < rows.length; ++i) {
@ -1494,7 +1577,7 @@ class PGClient extends KnexClient {
`SELECT format('%I.%I(%s)', ns.nspname, p.proname, oidvectortypes(p.proargtypes)) as function_declaration, pg_get_functiondef(p.oid) as create_function
FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = ? and p.proname = ?;`,
[this.schema, args.function_name],
[this.getEffectiveSchema(args), args.function_name],
);
// log.debug(response);
@ -2229,8 +2312,12 @@ class PGClient extends KnexClient {
for (let i = 0; i < args.columns.length; i++) {
const column = args.columns[i];
if (column.au) {
const triggerFnName = `xc_au_${args.tn}_${column.cn}`;
const triggerName = `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnName = args.schema
? `xc_au_${args.schema}_${args.tn}_${column.cn}`
: `xc_au_${args.tn}_${column.cn}`;
const triggerName = args.schema
? `xc_trigger_${args.schema}_${args.tn}_${column.cn}`
: `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnQuery = this.genQuery(
`CREATE OR REPLACE FUNCTION ??()
@ -2252,14 +2339,18 @@ class PGClient extends KnexClient {
BEFORE UPDATE ON ??
FOR EACH ROW
EXECUTE PROCEDURE ??();`,
[triggerName, args.tn, triggerFnName],
[
triggerName,
args.schema ? `${args.schema}.${args.tn}` : args.tn,
triggerFnName,
],
);
downQuery +=
this.querySeparator() +
this.genQuery(`DROP TRIGGER IF EXISTS ?? ON ??;`, [
triggerName,
args.tn,
args.schema ? `${args.schema}.${args.tn}` : args.tn,
]) +
this.querySeparator() +
this.genQuery(`DROP FUNCTION IF EXISTS ??()`, [triggerFnName]);
@ -2280,8 +2371,12 @@ class PGClient extends KnexClient {
for (let i = 0; i < args.columns.length; i++) {
const column = args.columns[i];
if (column.au && column.altered === 1) {
const triggerFnName = `xc_au_${args.tn}_${column.cn}`;
const triggerName = `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnName = args.schema
? `xc_au_${args.schema}_${args.tn}_${column.cn}`
: `xc_au_${args.tn}_${column.cn}`;
const triggerName = args.schema
? `xc_trigger_${args.schema}_${args.tn}_${column.cn}`
: `xc_trigger_${args.tn}_${column.cn}`;
const triggerFnQuery = this.genQuery(
`CREATE OR REPLACE FUNCTION ??()
@ -2303,7 +2398,11 @@ class PGClient extends KnexClient {
BEFORE UPDATE ON ??
FOR EACH ROW
EXECUTE PROCEDURE ??();`,
[triggerName, args.tn, triggerFnName],
[
triggerName,
args.schema ? `${args.schema}.${args.tn}` : args.tn,
triggerFnName,
],
);
downQuery +=
@ -2356,7 +2455,7 @@ class PGClient extends KnexClient {
log.api(`${_func}:args:`, args);
try {
args.table = args.tn;
args.table = args.schema ? `${args.schema}.${args.tn}` : args.tn;
const originalColumns = args.originalColumns;
args.connectionConfig = this._connectionConfig;
args.sqlClient = this.sqlClient;
@ -2477,7 +2576,9 @@ class PGClient extends KnexClient {
/** ************** create up & down statements *************** */
const upStatement =
this.querySeparator() +
this.sqlClient.schema.dropTable(args.tn).toString();
this.sqlClient.schema
.dropTable(args.schema ? `${args.schema}.${args.tn}` : args.tn)
.toString();
let downQuery = this.createTable(args.tn, args);
/**
@ -2563,7 +2664,9 @@ class PGClient extends KnexClient {
/** ************** drop tn *************** */
await this.sqlClient.raw(
this.sqlClient.schema.dropTable(args.tn).toQuery(),
this.sqlClient.schema
.dropTable(args.schema ? `${args.schema}.${args.tn}` : args.tn)
.toQuery(),
);
/** ************** return files *************** */
@ -2838,7 +2941,9 @@ class PGClient extends KnexClient {
query += this.alterTablePK(table, args.columns, [], query, true);
query = this.genQuery(`CREATE TABLE ?? (${query});`, [args.tn]);
query = this.genQuery(`CREATE TABLE ?? (${query});`, [
args.schema ? `${args.schema}.${args.tn}` : args.tn,
]);
return query;
}
@ -2954,12 +3059,7 @@ class PGClient extends KnexClient {
}
get schema() {
return (
(this.connectionConfig &&
this.connectionConfig.searchPath &&
this.connectionConfig.searchPath[0]) ||
'public'
);
return this.connectionConfig?.searchPath?.[0] || 'public';
}
/**
@ -3025,7 +3125,10 @@ class PGClient extends KnexClient {
await this.sqlClient.raw(
this.sqlClient.schema
.renameTable(
this.sqlClient.raw('??.??', [this.schema, args.tn_old]),
this.sqlClient.raw('??.??', [
this.getEffectiveSchema(args),
args.tn_old,
]),
args.tn,
)
.toQuery(),
@ -3036,7 +3139,10 @@ class PGClient extends KnexClient {
this.querySeparator() +
this.sqlClient.schema
.renameTable(
this.sqlClient.raw('??.??', [this.schema, args.tn]),
this.sqlClient.raw('??.??', [
this.getEffectiveSchema(args),
args.tn,
]),
args.tn_old,
)
.toQuery();
@ -3047,7 +3153,10 @@ class PGClient extends KnexClient {
this.querySeparator() +
this.sqlClient.schema
.renameTable(
this.sqlClient.raw('??.??', [this.schema, args.tn_old]),
this.sqlClient.raw('??.??', [
this.getEffectiveSchema(args),
args.tn_old,
]),
args.tn,
)
.toQuery();
@ -3064,6 +3173,151 @@ class PGClient extends KnexClient {
return result;
}
/**
*
* @param {Object} - args
* @param {String} - args.tn
* @param {String} - args.indexName
* @param {String} - args.non_unique
* @param {String[]} - args.columns
* @returns {Promise<{upStatement, downStatement}>}
*/
async indexCreate(args) {
const _func = this.indexCreate.name;
const result = new Result();
log.api(`${_func}:args:`, args);
const indexName = args.indexName || null;
try {
args.table = args.schema ? `${args.schema}.${args.tn}` : args.tn;
// s = await this.sqlClient.schema.index(Object.keys(args.columns));
await this.sqlClient.raw(
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique) {
table.index(args.columns, indexName);
} else {
table.unique(args.columns, indexName);
}
})
.toQuery(),
);
const upStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique) {
table.index(args.columns, indexName);
} else {
table.unique(args.columns, indexName);
}
})
.toQuery();
this.emit(`Success : ${upStatement}`);
const downStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique) {
table.dropIndex(args.columns, indexName);
} else {
table.dropUnique(args.columns, indexName);
}
})
.toQuery();
result.data.object = {
upStatement: [{ sql: upStatement }],
downStatement: [{ sql: downStatement }],
};
// result.data.object = {
// upStatement,
// downStatement
// };
} catch (e) {
log.ppe(e, _func);
throw e;
}
return result;
}
/**
*
* @param {Object} - args
* @param {String} - args.tn
* @param {String[]} - args.columns
* @param {String} - args.indexName
* @param {String} - args.non_unique
* @returns {Promise<{upStatement, downStatement}>}
*/
async indexDelete(args) {
const _func = this.indexDelete.name;
const result = new Result();
log.api(`${_func}:args:`, args);
const indexName = args.indexName || null;
try {
args.table = args.schema ? `${args.schema}.${args.tn}` : args.tn;
// s = await this.sqlClient.schema.index(Object.keys(args.columns));
await this.sqlClient.raw(
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique_original) {
table.dropIndex(args.columns, indexName);
} else {
table.dropUnique(args.columns, indexName);
}
})
.toQuery(),
);
const upStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique_original) {
table.dropIndex(args.columns, indexName);
} else {
table.dropUnique(args.columns, indexName);
}
})
.toQuery();
this.emit(`Success : ${upStatement}`);
const downStatement =
this.querySeparator() +
this.sqlClient.schema
.table(args.table, function (table) {
if (args.non_unique_original) {
table.index(args.columns, indexName);
} else {
table.unique(args.columns, indexName);
}
})
.toQuery();
result.data.object = {
upStatement: [{ sql: upStatement }],
downStatement: [{ sql: downStatement }],
};
} catch (e) {
log.ppe(e, _func);
throw e;
}
return result;
}
}
export default PGClient;

57
packages/nocodb/src/helpers/initBaseBehaviour.ts

@ -0,0 +1,57 @@
import { Logger } from '@nestjs/common';
import { Knex } from 'knex';
import PgConnectionConfig = Knex.PgConnectionConfig;
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import CustomKnex from '~/db/CustomKnex';
const logger = new Logger('initBaseBehavior');
export async function initBaseBehavior() {
const dataConfig = await NcConnectionMgrv2.getDataConfig();
// return if client is not postgres
if (dataConfig.client !== 'pg') {
return;
}
// if NC_MINIMAL_DBS already exists, return
if (process.env.NC_MINIMAL_DBS === 'false') {
return;
}
let tempConnection: Knex<any, unknown[]> | undefined;
try {
tempConnection = CustomKnex(dataConfig);
// check if database user have permission to create new schema
const schemaCreateAllowed = await tempConnection.raw(
"SELECT has_database_privilege(:user, :database, 'CREATE') as has_database_privilege",
{
database: (dataConfig.connection as PgConnectionConfig).database,
user: (dataConfig.connection as PgConnectionConfig).user,
},
);
// if schema creation is not allowed, return
if (!schemaCreateAllowed.rows[0]?.has_database_privilege) {
// set NC_MINIMAL_DBS to false if it's set to true and log warning
if (process.env.NC_MINIMAL_DBS === 'true') {
process.env.NC_MINIMAL_DBS = 'false';
}
logger.warn(
`User ${(dataConfig.connection as PgConnectionConfig)?.user} does not have permission to create schema, minimal databases feature will be disabled`,
);
return;
}
// set NC_MINIMAL_DBS to true
process.env.NC_MINIMAL_DBS = 'true';
} catch (error) {
logger.warn(
`Error while checking schema creation permission: ${error.message}`,
);
} finally {
// close the connection since it's only used to verify permission
await tempConnection?.destroy();
}
}

4
packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts

@ -50,6 +50,7 @@ import * as nc_060_descriptions from '~/meta/migrations/v2/nc_060_descriptions';
import * as nc_061_integration_is_default from '~/meta/migrations/v2/nc_061_integration_is_default';
import * as nc_062_integration_store from '~/meta/migrations/v2/nc_062_integration_store';
import * as nc_063_form_field_filter from '~/meta/migrations/v2/nc_063_form_field_filter';
import * as nc_064_pg_minimal_dbs from '~/meta/migrations/v2/nc_064_pg_minimal_dbs';
// Create a custom migration source class
export default class XcMigrationSourcev2 {
@ -111,6 +112,7 @@ export default class XcMigrationSourcev2 {
'nc_061_integration_is_default',
'nc_062_integration_store',
'nc_063_form_field_filter',
'nc_064_pg_minimal_dbs',
]);
}
@ -224,6 +226,8 @@ export default class XcMigrationSourcev2 {
return nc_062_integration_store;
case 'nc_063_form_field_filter':
return nc_063_form_field_filter;
case 'nc_064_pg_minimal_dbs':
return nc_064_pg_minimal_dbs;
}
}
}

16
packages/nocodb/src/meta/migrations/v2/nc_064_pg_minimal_dbs.ts

@ -0,0 +1,16 @@
import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => {
table.boolean('is_local').defaultTo(false);
});
};
const down = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => {
table.dropColumn('is_local');
});
};
export { up, down };

13
packages/nocodb/src/models/Model.ts

@ -34,6 +34,7 @@ import {
prepareForDb,
prepareForResponse,
} from '~/utils/modelUtils';
import { Source } from '~/models';
const logger = new Logger('Model');
@ -471,21 +472,33 @@ export default class Model implements TableType {
dbDriver: XKnex;
model?: Model;
extractDefaultView?: boolean;
source?: Source;
},
ncMeta = Noco.ncMeta,
): Promise<BaseModelSqlv2> {
const model = args?.model || (await this.get(context, args.id, ncMeta));
const source =
args.source ||
(await Source.get(context, model.source_id, false, ncMeta));
if (!args?.viewId && args.extractDefaultView) {
const view = await View.getDefaultView(context, model.id, ncMeta);
args.viewId = view.id;
}
let schema: string;
if (source?.isMeta(true, 1)) {
schema = source.getConfig()?.schema;
} else if (source?.type === 'pg') {
schema = source.getConfig()?.searchPath?.[0];
}
return new BaseModelSqlv2({
context,
dbDriver: args.dbDriver,
viewId: args.viewId,
model,
schema,
});
}

17
packages/nocodb/src/models/Source.ts

@ -34,6 +34,7 @@ export default class Source implements SourceType {
alias?: string;
type?: DriverClient;
is_meta?: BoolType;
is_local?: BoolType;
is_schema_readonly?: BoolType;
is_data_readonly?: BoolType;
config?: string;
@ -71,6 +72,7 @@ export default class Source implements SourceType {
'config',
'type',
'is_meta',
'is_local',
'inflection_column',
'inflection_table',
'order',
@ -131,6 +133,7 @@ export default class Source implements SourceType {
'config',
'type',
'is_meta',
'is_local',
'inflection_column',
'inflection_table',
'order',
@ -297,6 +300,15 @@ export default class Source implements SourceType {
}
public async getConnectionConfig(): Promise<any> {
if (this.is_meta || this.is_local) {
const metaConfig = await NcConnectionMgrv2.getDataConfig();
const config = { ...metaConfig };
if (config.client === 'sqlite3') {
config.connection = metaConfig;
}
return config;
}
const config = this.getConfig();
// todo: update sql-client args
@ -307,7 +319,6 @@ export default class Source implements SourceType {
return config;
}
public getConfig(skipIntegrationConfig = false): any {
if (this.is_meta) {
const metaConfig = Noco.getConfig()?.meta?.db;
@ -556,9 +567,9 @@ export default class Source implements SourceType {
if (_mode === 0) {
return this.is_meta;
}
return false;
return this.is_local;
} else {
return this.is_meta;
return this.is_meta || this.is_local;
}
}

7
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@ -466,12 +466,7 @@ export class AtImportProcessor {
(value as any).name = 'nc_empty';
}
// skip duplicates (we don't allow them)
if (
options.find(
(el) =>
el.title === (value as any).name,
)
) {
if (options.find((el) => el.title === (value as any).name)) {
logWarning(
`Duplicate select option found: ${col.name} :: ${
(value as any).name

1
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

@ -244,7 +244,6 @@ async function readView(viewId) {
});
});
if (data?.data) {
return { view: data.data };
} else {

4
packages/nocodb/src/providers/init-meta-service.provider.ts

@ -13,6 +13,7 @@ import { User } from '~/models';
import { NcConfig, prepareEnv } from '~/utils/nc-config';
import { MetaTable, RootScopes } from '~/utils/globals';
import { updateMigrationJobsState } from '~/helpers/migrationJobs';
import { initBaseBehavior } from '~/helpers/initBaseBehaviour';
export const InitMetaServiceProvider: FactoryProvider = {
// initialize app,
@ -83,6 +84,9 @@ export const InitMetaServiceProvider: FactoryProvider = {
Noco.config = config;
Noco.eventEmitter = eventEmitter;
// decide base behavior based on env and database permissions
await initBaseBehavior();
if (!instanceConfig) {
// bump to latest version for fresh install
await updateMigrationJobsState({

18
packages/nocodb/src/services/bases.service.ts

@ -163,6 +163,22 @@ export class BasesService {
baseBody.prefix = `nc_${ranId}__`;
baseBody.is_meta = true;
if (process.env.NC_MINIMAL_DBS === 'true') {
const dataConfig = await Noco.getConfig()?.meta?.db;
if (dataConfig?.client === 'pg') {
baseBody.prefix = '';
baseBody.sources = [
{
type: 'pg',
is_local: true,
is_meta: false,
config: {
schema: baseId,
},
inflection_column: 'camelize',
inflection_table: 'camelize',
},
];
} else {
// if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each base
// each file will be named as nc_<random_id>.db
const fs = require('fs');
@ -181,6 +197,7 @@ export class BasesService {
{
type: 'sqlite3',
is_meta: false,
is_local: true,
config: {
client: 'sqlite3',
connection: {
@ -195,6 +212,7 @@ export class BasesService {
inflection_table: 'camelize',
},
];
}
} else {
const db = Noco.getConfig().meta?.db;
baseBody.sources = [

7
packages/nocodb/src/services/data-alias-nested.service.ts

@ -32,6 +32,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -80,6 +81,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -123,6 +125,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -166,6 +169,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -208,6 +212,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -253,6 +258,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);
@ -300,6 +306,7 @@ export class DataAliasNestedService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const column = await getColumnByIdOrName(context, param.columnName, model);

2
packages/nocodb/src/services/data-table.service.ts

@ -52,6 +52,7 @@ export class DataTableService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const row = await baseModel.readByPk(param.rowId, false, param.query, {
@ -82,6 +83,7 @@ export class DataTableService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
if (view.type !== ViewTypes.GRID) {

23
packages/nocodb/src/services/datas.service.ts

@ -96,6 +96,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const countArgs: any = { ...param.query, throwErrorIfInvalidParams: true };
@ -124,6 +125,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.nestedInsert(param.body, null, param.cookie);
@ -145,6 +147,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.updateByPk(
@ -166,6 +169,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
// if xcdb base skip checking for LTAR
@ -209,6 +213,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
}));
const { ast, dependencyFields } = await getAst(context, {
@ -273,6 +278,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const args: any = { ...query };
@ -305,6 +311,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...query };
@ -342,6 +349,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const row = await baseModel.readByPk(param.rowId, false, param.query, {
getHiddenColumn: param.getHiddenColumn,
@ -366,6 +374,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.exist(param.rowId);
@ -403,6 +412,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -487,6 +497,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `${model.title}List`;
@ -551,6 +562,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = 'List';
@ -615,6 +627,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = 'List';
@ -679,6 +692,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = 'List';
@ -743,6 +757,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `${model.title}List`;
@ -797,6 +812,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -830,6 +846,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.insert(param.body, null, param.cookie);
@ -854,6 +871,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.updateByPk(
@ -882,6 +900,7 @@ export class DatasService {
const baseModel = await Model.getBaseModelSQL(context, {
id: model.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
return await baseModel.delByPk(param.rowId, null, param.cookie);
@ -911,6 +930,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
await baseModel.removeChild({
@ -947,6 +967,7 @@ export class DatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
await baseModel.addChild({
@ -1010,6 +1031,7 @@ export class DatasService {
id: view.model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { offset, dbRows, elapsed } = await getDbRows(context, {
@ -1048,6 +1070,7 @@ export class DatasService {
id: view.model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { offset, dbRows, elapsed } = await getDbRows(context, {

10
packages/nocodb/src/services/public-datas.service.ts

@ -65,6 +65,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -129,6 +130,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...param.query };
@ -198,6 +200,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast } = await getAst(context, { model, query: param.query, view });
@ -293,6 +296,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...query };
@ -350,6 +354,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
await view.getViewWithInfo(context);
@ -470,6 +475,7 @@ export class PublicDatasService {
id: model.id,
viewId: colOptions.fk_target_view_id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const { ast, dependencyFields } = await getAst(context, {
@ -558,6 +564,7 @@ export class PublicDatasService {
id: view.fk_model_id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `List`;
@ -637,6 +644,7 @@ export class PublicDatasService {
id: view.fk_model_id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const key = `List`;
@ -711,6 +719,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const row = await baseModel.readByPk(rowId, false, query);
@ -813,6 +822,7 @@ export class PublicDatasService {
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(source),
source,
});
const listArgs: any = { ...param.query };

4
packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

@ -111,4 +111,8 @@ export default class NcConnectionMgrv2 {
...(await source.getConnectionConfig()),
});
}
public static async getDataConfig?() {
return Noco.getConfig()?.meta?.db;
}
}

Loading…
Cancel
Save