feat: Postgres base as schema (#9591)

* refactor:  introduce `NC_DISABLE_BASE_AS_PG_SCHEMA`

* refactor: change env name and create db if not exist

* refactor: review comments

* chore: add npmignore file

Signed-off-by: Pranav C <pranavxc@gmail.com>

---------

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md

@@ -76,7 +76,7 @@ For production use cases, it is crucial to set all environment variables marked
 
 ## Product Configuration
 | Variable | Mandatory | Description                                                                                                                                                                                           | If Not Set                                 |
-| -------- | --------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
+| -------- | --------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
 | `DB_QUERY_LIMIT_DEFAULT` | No | Default pagination limit for data tables.                                                                                                                                                             | Defaults to `25`. Maximum is `100`         |
 | `DB_QUERY_LIMIT_GROUP_BY_GROUP` | No | Number of groups per page.                                                                                                                                                                            | Defaults to `10`.                          |
 | `DB_QUERY_LIMIT_GROUP_BY_RECORD` | No | Number of records per group.                                                                                                                                                                          | Defaults to `10`.                          |
@@ -89,19 +89,14 @@ For production use cases, it is crucial to set all environment variables marked
 | `NC_ALLOW_LOCAL_HOOKS` | No | Allows webhooks to call local network links, posing potential security risks. Set to `true` to enable; all other values are considered `false`.                                                       | Defaults to `false`.                       |
 | `NC_SANITIZE_COLUMN_NAME` | No | Enables sanitization of column names during their creation to prevent SQL injection and other security issues.                                                                                        | Defaults to `true`.                        |
 | `NC_TOOL_DIR` | No | Specifies the directory to store metadata and app-related files. In Docker setups, this maps to `/usr/app/data/` for mounting volumes.                                                                | Defaults to the current working directory. |
-| `NC_MINIMAL_DBS` | No | Enables the minimal database feature of NocoDB. For more details, see [Minimal Database behavior](#minimal-database). | Enabled by default for PostgreSQL when the database user has schema creation permission. Set to `false` to disable. |
+| `NC_DISABLE_BASE_AS_PG_SCHEMA` | No | Disables the creation of a schema for each base in PostgreSQL.  [Click here for more detail](#postgres-base-as-schema)                                                                                |                                            |
 | `NC_MIGRATIONS_DISABLED` | No | Disables NocoDB migrations.                                                                                                                                                                           |                                            |
 | `NC_DISABLE_AUDIT` | No | Disables the audit log feature.                                                                                                                                                                       | Defaults to `false`.                       |
 | `NC_AUTOMATION_LOG_LEVEL` | No | Configures logging levels for automation features. Possible values: `OFF`, `ERROR`, `ALL`. More details can be found under [Webhooks](/automation/webhook/create-webhook).                            | Defaults to `OFF`.                         |
 
-### Minimal Database
+### Postgres Base as Schema
 
-This feature organizes base data into smaller, modular structures for PostgreSQL and SQLite databases:
-
-- **SQLite**: Each base's data is stored as a separate SQLite file, ensuring isolated storage.
-- **PostgreSQL**: A unique schema is created for each base, providing logical separation within the database.
-
-**Note**: For PostgreSQL, this feature is enabled by default if the user has the required permissions. To disable it, set the `NC_MINIMAL_DBS` environment variable to `false`.
+For PostgreSQL, a unique schema is created for each base, providing logical separation within the database. This feature is enabled by default if the user has the required permissions. To disable it, set the `NC_DISABLE_BASE_AS_PG_SCHEMA` environment variable to `false`.
 
 
 ## Logging & Monitoring

packages/nocodb/.npmignore

@@ -0,0 +1,10 @@
+node_modules
+src
+tests
+test
+docker
+build-utils
+webpack.*.js
+tsconfig*.json
+Dockerfile*
+nest-cli.json

packages/nocodb/src/helpers/initBaseBehaviour.ts

@@ -3,8 +3,43 @@ import { Knex } from 'knex';
 import PgConnectionConfig = Knex.PgConnectionConfig;
 import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
 import CustomKnex from '~/db/CustomKnex';
+import { SqlClientFactory } from '~/db/sql-client/lib/SqlClientFactory';
 const logger = new Logger('initBaseBehavior');
 
+async function isSchemaCreateAllowed(
+  tempConnection: Knex<any, unknown[]>,
+  dataConfig: any,
+  skipDatabaseCreation = false,
+) {
+  try {
+    // check if database user have permission to create new schema
+    return await tempConnection.raw(
+      "SELECT has_database_privilege(:user, :database, 'CREATE') as has_database_privilege",
+      {
+        database: (dataConfig.connection as PgConnectionConfig).database,
+        user: (dataConfig.connection as PgConnectionConfig).user,
+      },
+    );
+  } catch (e) {
+    // check if error is due to database not found
+    if (e.message.includes('does not exist') && !skipDatabaseCreation) {
+      // create sqlClient from dataConfig
+      const sqlClient = SqlClientFactory.create({
+        ...dataConfig,
+        knex: tempConnection,
+      });
+
+      // create database if it doesn't exist
+      await sqlClient.createDatabaseIfNotExists({
+        database: dataConfig.connection.database,
+        schema: dataConfig.searchPath?.[0] || 'public',
+      });
+      return isSchemaCreateAllowed(tempConnection, dataConfig, true);
+    }
+    throw e;
+  }
+}
+
 export async function initBaseBehavior() {
   const dataConfig = await NcConnectionMgrv2.getDataConfig();
 
@@ -13,8 +48,8 @@ export async function initBaseBehavior() {
     return;
   }
 
-  // if NC_MINIMAL_DBS already exists, return
-  if (process.env.NC_MINIMAL_DBS === 'false') {
+  // disable minimal databases feature if NC_DISABLE_BASE_AS_PG_SCHEMA is set to true
+  if (process.env.NC_DISABLE_BASE_AS_PG_SCHEMA === 'true') {
     return;
   }
 
@@ -22,34 +57,30 @@ export async function initBaseBehavior() {
 
   try {
     tempConnection = CustomKnex(dataConfig);
-
-    // check if database user have permission to create new schema
-    const schemaCreateAllowed = await tempConnection.raw(
-      "SELECT has_database_privilege(:user, :database, 'CREATE') as has_database_privilege",
-      {
-        database: (dataConfig.connection as PgConnectionConfig).database,
-        user: (dataConfig.connection as PgConnectionConfig).user,
-      },
+    const schemaCreateAllowed = await isSchemaCreateAllowed(
+      tempConnection,
+      dataConfig,
     );
 
     // if schema creation is not allowed, return
-    if (!schemaCreateAllowed.rows[0]?.has_database_privilege) {
+    if (!schemaCreateAllowed?.rows?.[0]?.has_database_privilege) {
       // set NC_MINIMAL_DBS to false if it's set to true and log warning
-      if (process.env.NC_MINIMAL_DBS === 'true') {
-        process.env.NC_MINIMAL_DBS = 'false';
-      }
+      process.env.NC_DISABLE_BASE_AS_PG_SCHEMA = 'true';
       logger.warn(
-        `User ${(dataConfig.connection as PgConnectionConfig)?.user} does not have permission to create schema, minimal databases feature will be disabled`,
+        `User ${
+          (dataConfig.connection as PgConnectionConfig)?.user
+        } does not have permission to create schema, minimal databases feature will be disabled`,
       );
       return;
     }
 
     // set NC_MINIMAL_DBS to true
-    process.env.NC_MINIMAL_DBS = 'true';
+    process.env.NC_DISABLE_BASE_AS_PG_SCHEMA = 'false';
   } catch (error) {
     logger.warn(
       `Error while checking schema creation permission: ${error.message}`,
     );
+    process.env.NC_DISABLE_BASE_AS_PG_SCHEMA = 'true';
   } finally {
     // close the connection since it's only used to verify permission
     await tempConnection?.destroy();

packages/nocodb/src/providers/init-meta-service.provider.ts

@@ -84,9 +84,6 @@ export const InitMetaServiceProvider: FactoryProvider = {
     Noco.config = config;
     Noco.eventEmitter = eventEmitter;
 
-    // decide base behavior based on env and database permissions
-    await initBaseBehavior();
-
     if (!instanceConfig) {
       // bump to latest version for fresh install
       await updateMigrationJobsState({
@@ -116,6 +113,9 @@ export const InitMetaServiceProvider: FactoryProvider = {
     });
     T.emit('evt_app_started', await User.count());
 
+    // decide base behavior based on env and database permissions
+    await initBaseBehavior();
+
     return metaService;
   },
   provide: MetaService,

packages/nocodb/src/services/bases.service.ts

@@ -162,9 +162,12 @@ export class BasesService {
       const ranId = nanoid();
       baseBody.prefix = `nc_${ranId}__`;
       baseBody.is_meta = true;
-      if (process.env.NC_MINIMAL_DBS === 'true') {
       const dataConfig = await Noco.getConfig()?.meta?.db;
-        if (dataConfig?.client === 'pg') {
+
+      if (
+        dataConfig?.client === 'pg' &&
+        process.env.NC_DISABLE_BASE_AS_PG_SCHEMA !== 'true'
+      ) {
         baseBody.prefix = '';
         baseBody.sources = [
           {
@@ -178,7 +181,10 @@ export class BasesService {
             inflection_table: 'camelize',
           },
         ];
-        } else {
+      } else if (
+        dataConfig?.client === 'sqlite3' &&
+        process.env.NC_MINIMAL_DBS === 'true'
+      ) {
         // if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each base
         // each file will be named as nc_<random_id>.db
         const fs = require('fs');
@@ -212,7 +218,6 @@ export class BasesService {
             inflection_table: 'camelize',
           },
         ];
-        }
       } else {
         const db = Noco.getConfig().meta?.db;
         baseBody.sources = [