Browse Source

Merge pull request #9499 from nocodb/nc-refactor/remove-default-config-encryption

Nc refactor/remove default datasource config encryption
pull/9604/head
Pranav C 2 months ago committed by GitHub
parent
commit
c84bd6d486
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 62
      .github/workflows/jest-unit-test.yml
  2. 156
      .github/workflows/release-secret-cli.yml
  3. 2
      .gitignore
  4. 65
      packages/nc-gui/components/smartsheet/details/Fields.vue
  5. 5
      packages/nc-secret-mgr/.npmignore
  6. 3
      packages/nc-secret-mgr/dist/cli.js
  7. 68
      packages/nc-secret-mgr/dist/cli.js.LICENSE.txt
  8. 50
      packages/nc-secret-mgr/package.json
  9. 97
      packages/nc-secret-mgr/src/core/NcConfig.ts
  10. 3
      packages/nc-secret-mgr/src/core/NcError.ts
  11. 138
      packages/nc-secret-mgr/src/core/SecretManager.ts
  12. 4
      packages/nc-secret-mgr/src/core/index.ts
  13. 25
      packages/nc-secret-mgr/src/core/logger.ts
  14. 17
      packages/nc-secret-mgr/src/index.spec.ts
  15. 72
      packages/nc-secret-mgr/src/index.ts
  16. 24
      packages/nc-secret-mgr/src/nocodb/cli.js
  17. 16
      packages/nc-secret-mgr/tsconfig.json
  18. 46
      packages/nc-secret-mgr/webpack.config.js
  19. 3
      packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md
  20. 61
      packages/noco-docs/docs/100.data-sources/050.updating-secret.md
  21. 41
      packages/nocodb/jest.config.js
  22. 23
      packages/nocodb/package.json
  23. 9
      packages/nocodb/src/cli.ts
  24. 5
      packages/nocodb/src/controllers/api-tokens.controller.ts
  25. 2
      packages/nocodb/src/controllers/base-users.controller.spec.ts
  26. 5
      packages/nocodb/src/controllers/org-tokens.controller.ts
  27. 114
      packages/nocodb/src/helpers/initDataSourceEncryption.ts
  28. 2
      packages/nocodb/src/meta/meta.service.ts
  29. 4
      packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
  30. 8
      packages/nocodb/src/meta/migrations/v2/nc_037_rename_project_and_base.ts
  31. 4
      packages/nocodb/src/meta/migrations/v2/nc_042_integrations.ts
  32. 4
      packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts
  33. 4
      packages/nocodb/src/meta/migrations/v2/nc_051_source_readonly_columns.ts
  34. 4
      packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts
  35. 18
      packages/nocodb/src/meta/migrations/v2/nc_056_integration.ts
  36. 4
      packages/nocodb/src/meta/migrations/v2/nc_064_pg_minimal_dbs.ts
  37. 22
      packages/nocodb/src/meta/migrations/v2/nc_065_encrypt_flag.ts
  38. 257
      packages/nocodb/src/models/Integration.spec.ts
  39. 64
      packages/nocodb/src/models/Integration.ts
  40. 114
      packages/nocodb/src/models/Source.ts
  41. 1
      packages/nocodb/src/models/UserRefreshToken.ts
  42. 6
      packages/nocodb/src/providers/init-meta-service.provider.ts
  43. 16
      packages/nocodb/src/services/base-users/base-users.service.spec.ts
  44. 6
      packages/nocodb/src/services/integrations.service.ts
  45. 15
      packages/nocodb/src/services/org-users.service.spec.ts
  46. 0
      packages/nocodb/src/services/users/index.ts
  47. 35
      packages/nocodb/src/services/users/users.service.spec.ts
  48. 54
      packages/nocodb/src/utils/encryptDecrypt.ts
  49. 6
      packages/nocodb/src/utils/globals.ts
  50. 1
      packages/nocodb/src/utils/index.ts
  51. 19
      packages/nocodb/src/utils/nc-config/helpers.ts
  52. 24
      packages/nocodb/src/version-upgrader/NcUpgrader.ts
  53. 2
      packages/nocodb/src/version-upgrader/upgraders/0100002_ncFilterUpgrader.ts
  54. 6
      packages/nocodb/src/version-upgrader/upgraders/0101002_ncAttachmentUpgrader.ts
  55. 6
      packages/nocodb/src/version-upgrader/upgraders/0104002_ncAttachmentUpgrader.ts
  56. 2
      packages/nocodb/src/version-upgrader/upgraders/0104004_ncFilterUpgrader.ts
  57. 2
      packages/nocodb/src/version-upgrader/upgraders/0105002_ncStickyColumnUpgrader.ts
  58. 2
      packages/nocodb/src/version-upgrader/upgraders/0105003_ncFilterUpgrader.ts
  59. 2
      packages/nocodb/src/version-upgrader/upgraders/0105004_ncHookUpgrader.ts
  60. 4
      packages/nocodb/src/version-upgrader/upgraders/0107004_ncProjectConfigUpgrader.ts
  61. 4
      packages/nocodb/src/version-upgrader/upgraders/0108002_ncXcdbLTARUpgrader.ts
  62. 4
      packages/nocodb/src/version-upgrader/upgraders/0111002_ncXcdbLTARIndexUpgrader.ts
  63. 4
      packages/nocodb/src/version-upgrader/upgraders/0111005_ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts
  64. 119
      packages/nocodb/src/version-upgrader/upgraders/0225002_ncDatasourceDecrypt.ts
  65. 61
      packages/nocodb/webpack.cli.config.js
  66. 1070
      pnpm-lock.yaml
  67. 3
      pnpm-workspace.yaml
  68. 14
      scripts/updateCliVersion.js
  69. 2
      tests/playwright/pages/Dashboard/Details/ErdPage.ts

62
.github/workflows/jest-unit-test.yml

@ -0,0 +1,62 @@
name: "NestJS Unit Test"
on:
push:
branches: [develop]
paths:
- "packages/nocodb/**"
- ".github/workflows/jest-unit-test.yml"
pull_request:
types: [opened, reopened, synchronize, ready_for_review, labeled]
branches: [develop]
paths:
- "packages/nocodb/**"
- ".github/workflows/jest-unit-test.yml"
workflow_call:
# Triggered manually
workflow_dispatch:
jobs:
jest-unit-test:
runs-on: [self-hosted, aws]
timeout-minutes: 20
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft || inputs.force == true }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: 18.19.1
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 8
- name: Get pnpm store directory
shell: bash
timeout-minutes: 1
run: |
echo "STORE_PATH=/root/setup-pnpm/node_modules/.bin/store/v3" >> $GITHUB_ENV
- uses: actions/cache@v3
name: Setup pnpm cache
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-store-
- name: Set CI env
run: export CI=true
- name: Set NC Edition
run: export EE=true
- name: remove use-node-version line from .npmrc
run: sed -i '/^use-node-version/d' .npmrc
- name: install dependencies
run: pnpm bootstrap
- name: build nocodb-sdk
working-directory: ./packages/nocodb-sdk
run: |
pnpm run generate:sdk
pnpm run build:main
- name: run unit tests
working-directory: ./packages/nocodb
run: pnpm run test

156
.github/workflows/release-secret-cli.yml

@ -0,0 +1,156 @@
name: "Release : Secret CLI Executables"
on:
# Triggered manually
workflow_dispatch:
inputs:
tag:
description: "Tag name"
required: true
secrets:
NC_GITHUB_TOKEN:
required: true
jobs:
build-and-publish:
runs-on: [self-hosted, aws]
steps:
- uses: actions/checkout@v3
- name: Cache node modules
id: cache-npm
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
# npm cache files are stored in `~/.npm` on Linux/macOS
path: ~/.npm
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Cache pkg modules
id: cache-pkg
uses: actions/cache@v3
env:
cache-name: cache-pkg
with:
# pkg cache files are stored in `~/.pkg-cache`
path: ~/.pkg-cache
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Build executables
run: |
pnpm bootstrap
cd ./packages/nocodb
pnpm run build:cli
cd ../nc-secret-mgr
targetVersion=${{ github.event.inputs.tag || inputs.tag }} node ../../scripts/updateVersion.js
pnpm run build && pnpm run publish
# for building images for all platforms these libraries are required in Linux
- name: Install QEMU and ldid
run: |
sudo apt update
# Install qemu
sudo apt install qemu binfmt-support qemu-user-static
# install ldid
git clone https://github.com/daeken/ldid.git
cd ./ldid
./make.sh
sudo cp ./ldid /usr/local/bin
- uses: actions/setup-node@v3
with:
node-version: 16
- name : Install nocodb, other dependencies and build executables
run: |
cd ./packages/nc-secret-mgr
# install npm dependendencies
pnpm i
# Build sqlite binaries for all platforms
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=win32 --fallback-to-build --target_arch=x64 --target_libc=unknown
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=win32 --fallback-to-build --target_arch=ia32 --target_libc=unknown
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=darwin --fallback-to-build --target_arch=x64 --target_libc=unknown
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=darwin --fallback-to-build --target_arch=arm64 --target_libc=unknown
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=linux --fallback-to-build --target_arch=x64 --target_libc=glibc
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=linux --fallback-to-build --target_arch=arm64 --target_libc=glibc
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=linux --fallback-to-build --target_arch=x64 --target_libc=musl
./node_modules/.bin/node-pre-gyp install --directory=./node_modules/sqlite3 --target_platform=linux --fallback-to-build --target_arch=arm64 --target_libc=musl
# clean up code to optimize size
npx modclean --patterns="default:*" --run
# build executables
npm run build
ls ./dist
# Move macOS executables for signing
mkdir ./mac-dist
mv ./dist/nc-secret-arm64 ./mac-dist/
mv ./dist/nc-secret-x64 ./mac-dist/
- name: Upload executables(except mac executables) to release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.NC_GITHUB_TOKEN }}
file: dist/**
tag: ${{ github.event.inputs.tag || inputs.tag }}
overwrite: true
file_glob: true
repo_name: nocodb/nc-secret-mgr
- uses: actions/upload-artifact@master
with:
name: ${{ github.event.inputs.tag || inputs.tag }}
path: scripts/pkg-executable/mac-dist
retention-days: 1
sign-mac-executables:
runs-on: macos-latest
needs: build-executables
steps:
- uses: actions/download-artifact@master
with:
name: ${{ github.event.inputs.tag || inputs.tag }}
path: scripts/pkg-executable/mac-dist
- name: Sign macOS executables
run: |
/usr/bin/codesign --force -s - ./scripts/pkg-executable/mac-dist/nc-secret-arm64 -v
/usr/bin/codesign --force -s - ./scripts/pkg-executable/mac-dist/nc-secret-x64 -v
- uses: actions/upload-artifact@master
with:
name: ${{ github.event.inputs.tag || inputs.tag }}
path: scripts/pkg-executable/mac-dist
retention-days: 1
publish-mac-executables:
needs: [sign-mac-executables,build-executables]
runs-on: [self-hosted, aws]
steps:
- uses: actions/download-artifact@master
with:
name: ${{ github.event.inputs.tag || inputs.tag }}
path: scripts/pkg-executable/mac-dist
- name: Upload executables(except mac executables) to release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.NC_GITHUB_TOKEN }}
file: mac-dist/**
tag: ${{ github.event.inputs.tag || inputs.tag }}
overwrite: true
file_glob: true
repo_name: nocodb/nc-secret-mgr

2
.gitignore vendored

@ -96,3 +96,5 @@ test_noco.db
httpbin httpbin
.run/test-debug.run.xml .run/test-debug.run.xml
/packages/nc-secret-mgr/dist/index.js
/packages/nc-secret-mgr/dist/index.js.map

65
packages/nc-gui/components/smartsheet/details/Fields.vue

@ -341,8 +341,12 @@ const onFieldUpdate = (state: TableExplorerColumn, skipLinkChecks = false) => {
const diffs = Object.fromEntries( const diffs = Object.fromEntries(
Object.entries(pdiffs).filter(([_, value]) => value !== undefined), Object.entries(pdiffs).filter(([_, value]) => value !== undefined),
) as Partial<TableExplorerColumn> ) as Partial<TableExplorerColumn>
if (
if (Object.keys(diffs).length === 0 || (Object.keys(diffs).length === 1 && 'altered' in diffs)) { Object.keys(diffs).length === 0 ||
// skip custom prop since it's only used for custom LTAR links
(Object.keys(diffs).length === 1 && 'custom' in diffs && Object.keys(diffs.custom).length === 0) ||
(Object.keys(diffs).length === 1 && 'altered' in diffs)
) {
ops.value = ops.value.filter((op) => op.op === 'add' || !compareCols(op.column, state)) ops.value = ops.value.filter((op) => op.op === 'add' || !compareCols(op.column, state))
} else { } else {
const field = ops.value.find((op) => compareCols(op.column, state)) const field = ops.value.find((op) => compareCols(op.column, state))
@ -353,10 +357,13 @@ const onFieldUpdate = (state: TableExplorerColumn, skipLinkChecks = false) => {
newFields.value = newFields.value.map((op) => { newFields.value = newFields.value.map((op) => {
if (compareCols(op, state)) { if (compareCols(op, state)) {
ops.value = ops.value.filter((op) => op.op === 'add' && !compareCols(op.column, state)) ops.value = ops.value.filter((op) => op.op === 'add' && !compareCols(op.column, state))
ops.value.push({ ops.value = [
op: 'add', ...ops.value,
column: state, {
}) op: 'add',
column: state,
},
]
return state return state
} }
return op return op
@ -372,16 +379,22 @@ const onFieldUpdate = (state: TableExplorerColumn, skipLinkChecks = false) => {
('childViewId' in diffs && diffs.childViewId !== col.colOptions?.fk_target_view_id) || ('childViewId' in diffs && diffs.childViewId !== col.colOptions?.fk_target_view_id) ||
checkForFilterChange(diffs.filters || []) checkForFilterChange(diffs.filters || [])
) { ) {
ops.value.push({ ops.value = [
op: 'update', ...ops.value,
column: state, {
}) op: 'update',
column: state,
},
]
} }
} else { } else {
ops.value.push({ ops.value = [
op: 'update', ...ops.value,
column: state, {
}) op: 'update',
column: state,
},
]
} }
if ( if (
@ -411,10 +424,13 @@ const onFieldDelete = (state: TableExplorerColumn) => {
field.column = state field.column = state
} }
} else { } else {
ops.value.push({ ops.value = [
op: 'delete', ...ops.value,
column: state, {
}) op: 'delete',
column: state,
},
]
} }
} }
@ -426,11 +442,14 @@ const onFieldAdd = (state: TableExplorerColumn) => {
state.temp_id = `temp_${++temporaryAddCount.value}` state.temp_id = `temp_${++temporaryAddCount.value}`
state.view_id = view.value?.id as string state.view_id = view.value?.id as string
ops.value.push({ ops.value = [
op: 'add', ...ops.value,
column: state, {
}) op: 'add',
newFields.value.push(state) column: state,
},
]
newFields.value = [...newFields.value, state]
if (addFieldMoveHook.value) { if (addFieldMoveHook.value) {
moveOps.value.push({ moveOps.value.push({

5
packages/nc-secret-mgr/.npmignore

@ -0,0 +1,5 @@
tsconfig.json
webpack.config.js
src
node_modules
scripts

3
packages/nc-secret-mgr/dist/cli.js vendored

File diff suppressed because one or more lines are too long

68
packages/nc-secret-mgr/dist/cli.js.LICENSE.txt vendored

@ -0,0 +1,68 @@
/*!
* mime-db
* Copyright(c) 2014 Jonathan Ong
* Copyright(c) 2015-2022 Douglas Christopher Wilson
* MIT Licensed
*/
/*!
* mime-types
* Copyright(c) 2014 Jonathan Ong
* Copyright(c) 2015 Douglas Christopher Wilson
* MIT Licensed
*/
/*! *****************************************************************************
Copyright (C) Microsoft. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of the
License at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions
and limitations under the License.
***************************************************************************** */
/**
* @copyright Copyright (c) 2021, Xgene Cloud Ltd
*
* @author Naveen MR <oof1lab@gmail.com>
* @author Pranav C Balan <pranavxc@gmail.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
/** @preserve
* Counter block mode compatible with Dr Brian Gladman fileenc.c
* derived from CryptoJS.mode.CTR
* Jan Hruby jhruby.web@gmail.com
*/
/** @preserve
(c) 2012 by Cédric Mesnil. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

50
packages/nc-secret-mgr/package.json

@ -0,0 +1,50 @@
{
"name": "nc-secret",
"version": "0.0.1",
"description": "",
"main": "dist/cli.js",
"bin": "dist/cli.js",
"scripts": {
"build": "webpack --config webpack.config.js",
"dev": "cross-env NC_DB=\"pg://localhost:5432?u=postgres&p=password&d=meta_2024_09_07\" nodemon --watch 'src/**/*.ts' --exec 'ts-node --project tsconfig.json' src/index.ts -- a b --nc-db abc",
"test": "mocha --require ts-node/register src/**/*.spec.ts",
"build:pkg": "npx pkg . --out-path dist --compress GZip",
"publish": "npm publish ."
},
"pkg": {
"assets": [
"node_modules/**/*"
],
"targets": [
"node16-linux-arm64",
"node16-macos-arm64",
"node16-win-arm64",
"node16-linux-x64",
"node16-macos-x64",
"node16-win-x64"
]
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"chalk": "^5.3.0",
"commander": "^12.1.0",
"enquirer": "^2.4.1",
"figlet": "^1.7.0",
"knex": "^3.1.0",
"mysql": "^2.18.1",
"parse-database-url": "^0.3.0",
"pg": "^8.12.0",
"sqlite3": "^5.1.7"
},
"devDependencies": {
"@types/figlet": "^1.5.8",
"chai": "^4.4.1",
"class-transformer": "0.3.1",
"cross-env": "^7.0.3",
"mocha": "^10.3.0",
"nodemon": "^3.0.3",
"pkg": "^5.8.0"
}
}

97
packages/nc-secret-mgr/src/core/NcConfig.ts

@ -0,0 +1,97 @@
import * as path from 'path';
import fs from 'fs';
import { promisify } from 'util';
const { DriverClient, getToolDir, metaUrlToDbConfig, prepareEnv } = require( '../nocodb/cli');
export class NcConfig {
meta: {
db: any;
} = {
db: {
client: DriverClient.SQLITE,
connection: {
filename: 'noco.db',
},
},
};
toolDir: string;
private constructor() {
this.toolDir = getToolDir();
}
public static async create(param: {
meta: {
metaUrl?: string;
metaJson?: string;
metaJsonFile?: string;
databaseUrlFile?: string;
databaseUrl?: string;
};
secret?: string;
}): Promise<NcConfig> {
const { meta, secret } =
param;
const ncConfig = new NcConfig();
if (ncConfig.meta?.db?.connection?.filename) {
ncConfig.meta.db.connection.filename = path.join(
ncConfig.toolDir,
ncConfig.meta.db.connection.filename,
);
}
if (meta?.metaUrl) {
ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl);
} else if (meta?.metaJson) {
ncConfig.meta.db = JSON.parse(meta.metaJson);
} else if (meta?.metaJsonFile) {
if (!(await promisify(fs.exists)(meta.metaJsonFile))) {
throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`);
}
const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, {
encoding: 'utf8',
});
ncConfig.meta.db = JSON.parse(fileContent);
}
return ncConfig;
}
public static async createByEnv(): Promise<NcConfig> {
return NcConfig.create({
meta: {
metaUrl: process.env.NC_DB,
metaJson: process.env.NC_DB_JSON,
metaJsonFile: process.env.NC_DB_JSON_FILE,
},
secret: process.env.NC_AUTH_JWT_SECRET,
});
}
}
export const getNocoConfig = async (options: {
ncDb?: string;
ncDbJson?: string;
ncDbJsonFile?: string;
databaseUrl?: string;
databaseUrlFile?: string;
} ={}) =>{
// check for JDBC url specified in env or options
await prepareEnv({
databaseUrl: options.databaseUrl || process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
databaseUrlFile: options.databaseUrlFile || process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
})
// create NocoConfig using utility method which works similar to Nocodb NcConfig with only meta db config
return NcConfig.create({
meta: {
metaUrl: process.env.NC_DB || options.ncDb,
metaJson: process.env.NC_DB_JSON || options.ncDbJson,
metaJsonFile: process.env.NC_DB_JSON_FILE || options.ncDbJsonFile,
}
});
}

3
packages/nc-secret-mgr/src/core/NcError.ts

@ -0,0 +1,3 @@
export class NcError extends Error {
}

138
packages/nc-secret-mgr/src/core/SecretManager.ts

@ -0,0 +1,138 @@
import {NcError} from "./NcError";
import * as logger from "./logger";
const { SqlClientFactory, MetaTable, decryptPropIfRequired, encryptPropIfRequired } = require('../nocodb/cli')
export class SecretManager {
private sqlClient;
constructor(private prevSecret: string, private newSecret: string, private config: any) {
this.sqlClient = SqlClientFactory.create(this.config.meta.db);
}
// validate config by checking if database config is valid
async validateConfig() {
// if sqlite then check the file exist in provided path
if (this.config.meta.db.client === 'sqlite3') {
if (!existsSync(this.config.meta.db.connection.filename)) {
throw new NcError('SQLite database file not found at path: ' + this.config.meta.db.connection.filename);
}
}
// use the sqlClientFactory to create a new sql client and then use testConnection to test the connection
const isValid = await this.sqlClient.testConnection();
if (!isValid) {
throw new NcError('Invalid database configuration. Please verify your database settings and ensure the database is reachable.');
}
}
async validateAndExtract() {
// check if tables are present in the database
if (!(await this.sqlClient.knex.schema.hasTable(MetaTable.SOURCES))) {
throw new NcError('Sources table not found');
}
if (!(await this.sqlClient.knex.schema.hasTable(MetaTable.INTEGRATIONS))) {
throw new NcError('Integrations table not found');
}
// if is_encrypted column is not present in the sources table then throw an error
if(
!(await this.sqlClient.knex.schema.hasColumn(MetaTable.SOURCES, 'is_encrypted')) ||
!(await this.sqlClient.knex.schema.hasColumn(MetaTable.INTEGRATIONS, 'is_encrypted'))){
throw new NcError('Looks like you are using an older version of NocoDB. Please upgrade to the latest version and try again.');
}
const sources = await this.sqlClient.knex(MetaTable.SOURCES).where(qb => {
qb.where('is_meta', false).orWhere('is_meta', null)
});
const integrations = await this.sqlClient.knex(MetaTable.INTEGRATIONS);
const sourcesToUpdate: Record<string, any>[] = [];
const integrationsToUpdate: Record<string, any>[] = [];
let isValid = false;
for (const source of sources) {
try {
const decrypted = decryptPropIfRequired({
data: source,
secret: this.prevSecret,
prop: 'config'
});
isValid = true;
sourcesToUpdate.push({ ...source, config: decrypted });
} catch (e) {
logger.error('Failed to decrypt source configuration : ' + e.message);
}
}
for (const integration of integrations) {
try {
const decrypted = decryptPropIfRequired({
data: integration,
secret: this.prevSecret,
prop: 'config'
});
isValid = true;
integrationsToUpdate.push({ ...integration, config: decrypted });
} catch (e) {
console.log(e);
}
}
// If all decryptions have failed, then throw an error
if (!isValid) {
throw new NcError('Invalid old secret or no sources/integrations found');
}
return { sourcesToUpdate, integrationsToUpdate };
}
async updateSecret(
sourcesToUpdate: Record<string, any>[],
integrationsToUpdate: Record<string, any>[]
) {
// start transaction
const transaction = await this.sqlClient.transaction();
try {
// update sources
for (const source of sourcesToUpdate) {
await transaction(MetaTable.SOURCES).update({
config: encryptPropIfRequired({
data: source,
secret: this.newSecret,
prop: 'config'
})
}).where('id', source.id);
}
// update integrations
for (const integration of integrationsToUpdate) {
await transaction(MetaTable.INTEGRATIONS).update({
config: encryptPropIfRequired({
data: integration,
secret: this.newSecret,
prop: 'config'
})
}).where('id', integration.id);
}
await transaction.commit();
} catch (e) {
logger.error('Failed to decrypt integration configuration: ' + e.message);
await transaction.rollback();
throw e;
}
}
}

4
packages/nc-secret-mgr/src/core/index.ts

@ -0,0 +1,4 @@
export * from './NcConfig';
export * from './NcError';
export * as logger from './logger';
export * from './SecretManager';

25
packages/nc-secret-mgr/src/core/logger.ts

@ -0,0 +1,25 @@
import chalk from 'chalk';
export function log(message: string) {
console.log(chalk.white(message));
}
export function error(message: string) {
console.error(chalk.red('Error: ' + message));
}
export function warn(message: string) {
console.warn(chalk.yellow('Warning: ' + message));
}
export function info(message: string) {
console.info(chalk.green('Info: ' + message));
}
export function success(message: string) {
console.log(chalk.green('Success: ' + message));
}
export function debug(message: string) {
console.debug(chalk.blue('Debug: ' + message));
}

17
packages/nc-secret-mgr/src/index.spec.ts

@ -0,0 +1,17 @@
import { describe, it } from 'mocha';
import { expect } from 'chai';
import { program } from 'commander';
describe('Index', () => {
describe('index.ts', () => {
it('should parse the arguments and options correctly', () => {
const argv = ['node', 'index.ts', 'prevSecret', 'newSecret', '--nc-db','test_db_url', '--database-url', 'test_db_url', '-o', 'prevSecret', '-n', 'newSecret'];
program.parse(argv);
expect(program.opts().prev).to.equal('prevSecret');
expect(program.opts().new).to.equal('newSecret');
expect(program.opts().ncDb).to.equal('test_db_url');
expect(program.opts().databaseUrl).to.equal('test_db_url');
});
});
});

72
packages/nc-secret-mgr/src/index.ts

@ -0,0 +1,72 @@
import figlet from "figlet";
import { Command } from 'commander';
import { getNocoConfig } from "./core";
import { SecretManager } from "./core";
import { NcError } from "./core";
import { logger } from "./core";
console.log(figlet.textSync("NocoDB Secret CLI"));
const program = new Command();
program
.version('1.0.0')
.description('NocoDB Secret CLI')
.arguments('<prevSecret> <newSecret>')
.option('--nc-db <char>', 'NocoDB connection database url, equivalent to NC_DB env variable')
.option('--nc-db-json <char>', 'NocoDB connection database json, equivalent to NC_DB_JSON env variable')
.option('--nc-db-json-file <char>', 'NocoDB connection database json file path, equivalent to NC_DB_JSON_FILE env variable')
.option('--database-url <char>', 'JDBC database url, equivalent to DATABASE_URL env variable')
.option('--database-url-file <char>', 'JDBC database url file path, equivalent to DATABASE_URL_FILE env variable')
.option('-p, --prev <char>', 'old secret string to decrypt sources and integrations')
.option('-n, --new <char>', 'new secret string to encrypt sources and integrations')
.action(async (prevVal, newVal) => {
try {
// extract options
const options = program.opts();
const config = await getNocoConfig(options);
const { prevSecret = prevVal, newSecret = newVal } = program.opts();
if (!prevSecret || !newSecret) {
console.error('Error: Both prevSecret and newSecret are required.');
program.help();
} else {
const secretManager = new SecretManager(prevSecret, newSecret, config);
// validate meta db config which is resolved from env variables
await secretManager.validateConfig();
// validate old secret
const { sourcesToUpdate, integrationsToUpdate } = await secretManager.validateAndExtract();
// update sources and integrations
await secretManager.updateSecret(sourcesToUpdate, integrationsToUpdate);
}
} catch (e) {
if (e instanceof NcError) {
// print error message in a better way
logger.error(e.message);
process.exit(1);
}
console.error(e);
process.exit(1);
}
});
// Add error handling
program.exitOverride((err) => {
console.error(err.message);
process.exit(1);
});
program.parse(process.argv);

24
packages/nc-secret-mgr/src/nocodb/cli.js

File diff suppressed because one or more lines are too long

16
packages/nc-secret-mgr/tsconfig.json

@ -0,0 +1,16 @@
{
"compilerOptions": {
"rootDir": "src",
"outDir": "dist",
"strict": true,
"target": "es6",
"module": "commonjs",
"sourceMap": true,
"esModuleInterop": true,
"moduleResolution": "node",
"skipLibCheck": true,
"noImplicitAny": false,
"allowJs": true,
"experimentalDecorators": true
}
}

46
packages/nc-secret-mgr/webpack.config.js

@ -0,0 +1,46 @@
const nodeExternals = require('webpack-node-externals');
const TerserPlugin = require('terser-webpack-plugin');
const webpack = require('webpack');
const path = require('path');
module.exports = {
entry: './src/index.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true
}
},
},
],
},
optimization: {
minimize: true,
minimizer: [new TerserPlugin()],
nodeEnv: false
},
externals: [nodeExternals()],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
},
output: {
filename: 'cli.js',
path: path.resolve(__dirname, 'dist'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
// node: {
// fs: 'empty'
// },
plugins: [
new webpack.BannerPlugin({banner: "#! /usr/bin/env node", raw: true}),
],
target: 'node',
};

3
packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md

@ -15,11 +15,12 @@ For production use cases, it is crucial to set all environment variables marked
| `NC_DB_JSON_FILE` | No | A path to a knex connection JSON file can be used to specify the database connection, as an alternative to `NC_DB`. | | | `NC_DB_JSON_FILE` | No | A path to a knex connection JSON file can be used to specify the database connection, as an alternative to `NC_DB`. | |
| `DATABASE_URL` | No | A [JDBC URL string](https://jdbc.postgresql.org/documentation/use/#connecting-to-the-database) can be used for the database connection instead of `NC_DB`. | | | `DATABASE_URL` | No | A [JDBC URL string](https://jdbc.postgresql.org/documentation/use/#connecting-to-the-database) can be used for the database connection instead of `NC_DB`. | |
| `DATABASE_URL_FILE` | No | A path to a file containing a JDBC URL can be specified for the database connection as an alternative to `NC_DB`. | | | `DATABASE_URL_FILE` | No | A path to a file containing a JDBC URL can be specified for the database connection as an alternative to `NC_DB`. | |
| `NC_CONNECTION_ENCRYPT_KEY` | No | The key used to encrypt the credentials of external databases. <br/> **Warning:** Changing this variable may break the application. If you must change it, use the CLI as described in the [NocoDB Secret CLI documentation](/data-sources/updating-secret). | Keep connection credentials as plain text in the database if not set. |
## Authentication ## Authentication
| Variable | Mandatory | Description | If Not Set | | Variable | Mandatory | Description | If Not Set |
| -------- |-----------| ----------- | ---------- | | -------- |-----------| ----------- | ---------- |
| `NC_AUTH_JWT_SECRET` | Yes | This JWT secret is utilized for generating authentication tokens and encrypting credentials for external databases. | A random secret will be generated automatically. | | `NC_AUTH_JWT_SECRET` | Yes | This JWT secret is utilized for generating authentication tokens. | A random secret will be generated automatically. |
| `NC_JWT_EXPIRES_IN` | No | Specifies the expiration time for JWT tokens. | Defaults to `10h`. | | `NC_JWT_EXPIRES_IN` | No | Specifies the expiration time for JWT tokens. | Defaults to `10h`. |
| `NC_GOOGLE_CLIENT_ID` | No | Google client ID required to activate Google authentication. | | | `NC_GOOGLE_CLIENT_ID` | No | Google client ID required to activate Google authentication. | |
| `NC_GOOGLE_CLIENT_SECRET` | No | Google client secret required to activate Google authentication. | | | `NC_GOOGLE_CLIENT_SECRET` | No | Google client secret required to activate Google authentication. | |

61
packages/noco-docs/docs/100.data-sources/050.updating-secret.md

@ -0,0 +1,61 @@
---
title: 'Updating Secrets'
description: 'Learn how to update secrets in NocoDB using the nc-secret-mgr package.'
tags: ['Secrets', 'nc-secret-mgr', 'Update', 'Security']
keywords: ['NocoDB secrets', 'nc-secret-mgr', 'Update', 'Security']
---
## Updating Secrets
To update a secret in NocoDB, you can use the `nc-secret-mgr` package. Follow the steps below to update a secret:
### Using the Command Line Interface (CLI)
1. Install the `nc-secret-mgr` package if you haven't already. You can do this by running the following command in your terminal:
```bash
npm install -g nc-secret-mgr
```
2. Once the package is installed, you can update a secret by running the following command:
```bash
NC_DB="pg://host:port?u=user&p=password&d=database" nc-secret-mgr update --prev <previous-secret> --new <new-secret>
```
OR
```bash
NC_DB="pg://host:port?u=user&p=password&d=database" nc-secret-mgr <previous-secret> <new-secret>
```
Replace `<previous-secret>` with the name of the secret you used previously, and `<new-secret>` with the new value of the secret.
3. After running the command, the secret will be updated in NocoDB.
### Using Executables
Alternatively, you can use the `nc-secret-mgr` executable to update secrets.
1. Download the `nc-secret-mgr` executable from the [NocoDB website](https://github.com/nocodb/nc-secret-mgr/releases/latest).
2. Run the executable using the following command:
```bash
NC_DB="pg://host:port?u=user&p=password&d=database" ./nc-secret-macos-arm64 update --prev <previous-secret> --new <new-secret>
```
Replace `<previous-secret>` with the name of the secret you used previously, and `<new-secret>` with the new value of the secret.
3. After running the command, the secret will be updated in NocoDB.
Note: All environment variables are supported, including `NC_DB`, `NC_DB_JSON`, `NC_DB_JSON_FILE`, `DATABASE_URL`, and `DATABASE_URL_FILE`. You can use any of these variables to specify your database connection. Alternatively, you can use the following equivalent parameters.
| Environment Variable | CLI Parameter |
| --------------------- | -------------- |
| `NC_DB` | `--nc-db` |
| `NC_DB_JSON` | `--nc-db-json` |
| `NC_DB_JSON_FILE` | `--nc-db-json-file` |
| `DATABASE_URL` | `--database-url` |
| `DATABASE_URL_FILE` | `--database-url-file` |

41
packages/nocodb/jest.config.js

@ -0,0 +1,41 @@
// jest.config.js
// In the following statement, replace `./tsconfig` with the path to your `tsconfig` file
// which contains the path mapping (ie the `compilerOptions.paths` option):
module.exports = {
moduleFileExtensions: ['js', 'json', 'ts', 'node'],
rootDir: 'src',
testRegex: '(Integration|Source)\\.spec\\.ts$',
collectCoverageFrom: ['**/*.(t|j)s'],
coverageDirectory: '../coverage',
testEnvironment: 'node',
moduleNameMapper: {
'^src/(.*)$': [
'<rootDir>/$1',
// '<rootDir>/$1/index'
],
'^~/(.*)$': [
'<rootDir>/ee/$1',
'<rootDir>/$1',
// '<rootDir>/ee/$1/index',
// '<rootDir>/$1/index',
],
'^@/(.*)$': ['<rootDir>/ee/$1', '<rootDir>/$1'],
},
// [...]
// moduleNameMapper: pathsToModuleNameMapper(
// compilerOptions.paths /*, { prefix: '<rootDir>/' } */,
// ),
// modulePaths: [compilerOptions.baseUrl],
// moduleNameMapper: pathsToModuleNameMapper(compilerOptions.paths, {
// prefix: '<rootDir>/../',
// }),
transform: {
'^.+\\.ts$': [
'ts-jest',
{
tsconfig: 'tsconfig.json',
},
],
},
};

23
packages/nocodb/package.json

@ -27,14 +27,14 @@
"start": "pnpm run watch:run", "start": "pnpm run watch:run",
"start:prod": "node docker/main", "start:prod": "node docker/main",
"lint": "eslint \"src/**/*.ts\" --fix", "lint": "eslint \"src/**/*.ts\" --fix",
"test": "jest", "test": "jest --runInBand --forceExit",
"test:watch": "jest --watch", "test:watch": "jest --watch",
"test:cov": "jest --coverage", "test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json", "test:e2e": "jest --config ./test/jest-e2e.json",
"watch:run": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/docker --log-error --project tsconfig.json\"", "watch:run": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/docker --log-error --project tsconfig.json\"",
"watch:run:mysql": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunMysql --log-error --project tsconfig.json\"", "watch:run:mysql": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunMysql --log-error --project tsconfig.json\"",
"watch:run:pg": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunPG --log-error --project tsconfig.json\"", "watch:run:pg": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunPG --log-error --project tsconfig.json\"",
"watch:run:playwright:mysql": "rm -f ./test_noco.db; cross-env DB_TYPE=mysql NC_DB=\"mysql2://localhost:3306?u=root&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"", "watch:run:playwright:mysql": "rm -f ./test_noco.db; cross-env DB_TYPE=mysql NC_DB=\"mysql2://localhost:3306?u=root&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"",
"watch:run:playwright:pg": "rm -f ./test_noco.db; cross-env DB_TYPE=pg NC_DB=\"pg://localhost:5432?u=postgres&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"", "watch:run:playwright:pg": "rm -f ./test_noco.db; cross-env DB_TYPE=pg NC_DB=\"pg://localhost:5432?u=postgres&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"",
"watch:run:playwright": "rm -f ./test_*.db; cross-env DB_TYPE=sqlite DATABASE_URL=sqlite:./test_noco.db PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true NC_SNAPSHOT_WINDOW_SEC=3 nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"", "watch:run:playwright": "rm -f ./test_*.db; cross-env DB_TYPE=sqlite DATABASE_URL=sqlite:./test_noco.db PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true NC_SNAPSHOT_WINDOW_SEC=3 nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"",
@ -196,22 +196,5 @@
"tsconfig-paths": "^4.2.0", "tsconfig-paths": "^4.2.0",
"typescript": "^5.3.3", "typescript": "^5.3.3",
"webpack-cli": "^5.1.4" "webpack-cli": "^5.1.4"
},
"jest": {
"moduleFileExtensions": [
"js",
"json",
"ts"
],
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"collectCoverageFrom": [
"**/*.(t|j)s"
],
"coverageDirectory": "../coverage",
"testEnvironment": "node"
} }
} }

9
packages/nocodb/src/cli.ts

@ -0,0 +1,9 @@
export { SqlClientFactory } from '~/db/sql-client/lib/SqlClientFactory';
export { MetaTable } from '~/utils/globals';
export * from '~/utils/encryptDecrypt';
export {
getToolDir,
metaUrlToDbConfig,
prepareEnv,
} from '~/utils/nc-config/helpers';
export { DriverClient } from '~/utils/nc-config/constants';

5
packages/nocodb/src/controllers/api-tokens.controller.ts

@ -51,7 +51,10 @@ export class ApiTokensController {
'/api/v2/meta/bases/:baseId/api-tokens/:tokenId', '/api/v2/meta/bases/:baseId/api-tokens/:tokenId',
]) ])
@Acl('baseApiTokenDelete') @Acl('baseApiTokenDelete')
async apiTokenDelete(@Req() req: NcRequest, @Param('tokenId') tokenId: string) { async apiTokenDelete(
@Req() req: NcRequest,
@Param('tokenId') tokenId: string,
) {
return await this.apiTokensService.apiTokenDelete({ return await this.apiTokensService.apiTokenDelete({
tokenId, tokenId,
user: req['user'], user: req['user'],

2
packages/nocodb/src/controllers/base-users.controller.spec.ts

@ -1,7 +1,7 @@
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { BaseUsersService } from '../services/base-users/base-users.service';
import { BaseUsersController } from './base-users.controller'; import { BaseUsersController } from './base-users.controller';
import type { TestingModule } from '@nestjs/testing'; import type { TestingModule } from '@nestjs/testing';
import { BaseUsersService } from '~/services/base-users/base-users.service';
describe('BaseUsersController', () => { describe('BaseUsersController', () => {
let controller: BaseUsersController; let controller: BaseUsersController;

5
packages/nocodb/src/controllers/org-tokens.controller.ts

@ -61,7 +61,10 @@ export class OrgTokensController {
// allowedRoles: [OrgUserRoles.SUPER], // allowedRoles: [OrgUserRoles.SUPER],
blockApiTokenAccess: true, blockApiTokenAccess: true,
}) })
async apiTokenDelete(@Req() req: NcRequest, @Param('tokenId') tokenId: string) { async apiTokenDelete(
@Req() req: NcRequest,
@Param('tokenId') tokenId: string,
) {
await this.orgTokensService.apiTokenDelete({ await this.orgTokensService.apiTokenDelete({
tokenId, tokenId,
user: req['user'], user: req['user'],

114
packages/nocodb/src/helpers/initDataSourceEncryption.ts

@ -0,0 +1,114 @@
import { Logger } from '@nestjs/common';
import Noco from '~/Noco';
import { MetaTable, RootScopes } from '~/utils/globals';
import { encryptPropIfRequired } from '~/utils';
const logger = new Logger('initDataSourceEncryption');
export default async function initDataSourceEncryption(_ncMeta = Noco.ncMeta) {
// return if env is not set
if (!process.env.NC_CONNECTION_ENCRYPT_KEY) {
return;
}
const secret = process.env.NC_CONNECTION_ENCRYPT_KEY;
const ncMeta = await _ncMeta.startTransaction();
const successStatus: boolean[] = [];
try {
// if configured, check for any non-encrypted data source by checking is_encrypted flag
const sources = await ncMeta
.knex(MetaTable.SOURCES)
.where((qb) => {
qb.where('is_encrypted', false).orWhereNull('is_encrypted');
})
.whereNotNull('config');
for (const source of sources) {
// skip if no config
if (!source.config) {
continue;
}
// check if valid json, if not warn and skip
try {
JSON.parse(source.config);
} catch (e) {
console.error('Invalid JSON in integration config', source.alias);
successStatus.push(false);
continue;
}
// encrypt the data source
await ncMeta.metaUpdate(
source.fk_workspace_id,
source.base_id,
MetaTable.SOURCES,
{
config: encryptPropIfRequired({
data: source,
secret,
}),
is_encrypted: true,
},
source.id,
);
successStatus.push(true);
}
const integrations = await ncMeta
.knex(MetaTable.INTEGRATIONS)
.where((qb) => {
qb.where('is_encrypted', false).orWhereNull('is_encrypted');
})
.whereNotNull('config');
for (const integration of integrations) {
// skip if no config
if (!integrations.config) {
continue;
}
// check if valid json, if not warn and skip
try {
JSON.parse(integrations.config);
} catch (e) {
logger.warn('Invalid JSON in integration config', integration.title);
successStatus.push(false);
continue;
}
// encrypt the data source
await ncMeta.metaUpdate(
RootScopes.WORKSPACE,
RootScopes.WORKSPACE,
MetaTable.INTEGRATIONS,
{
config: encryptPropIfRequired({
data: integration,
secret,
}),
is_encrypted: true,
},
integration.id,
);
successStatus.push(true);
}
// if all failed, throw error
if (successStatus.length && successStatus.every((status) => !status)) {
// if all fails then rollback and exit
throw new Error(
'Failed to encrypt all data sources, please remove invalid data sources and try again.',
);
}
await ncMeta.commit();
} catch (e) {
await ncMeta.rollback();
console.error('Failed to encrypt data sources');
throw e;
}
}

2
packages/nocodb/src/meta/meta.service.ts

@ -292,7 +292,7 @@ export class MetaService {
public async genNanoid(target: string) { public async genNanoid(target: string) {
const prefixMap: { [key: string]: string } = { const prefixMap: { [key: string]: string } = {
[MetaTable.PROJECT]: 'p', [MetaTable.PROJECT]: 'p',
[MetaTable.BASES]: 'b', [MetaTable.SOURCES]: 'b',
[MetaTable.MODELS]: 'm', [MetaTable.MODELS]: 'm',
[MetaTable.COLUMNS]: 'c', [MetaTable.COLUMNS]: 'c',
[MetaTable.COL_RELATIONS]: 'l', [MetaTable.COL_RELATIONS]: 'l',

4
packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts

@ -51,6 +51,7 @@ import * as nc_061_integration_is_default from '~/meta/migrations/v2/nc_061_inte
import * as nc_062_integration_store from '~/meta/migrations/v2/nc_062_integration_store'; import * as nc_062_integration_store from '~/meta/migrations/v2/nc_062_integration_store';
import * as nc_063_form_field_filter from '~/meta/migrations/v2/nc_063_form_field_filter'; import * as nc_063_form_field_filter from '~/meta/migrations/v2/nc_063_form_field_filter';
import * as nc_064_pg_minimal_dbs from '~/meta/migrations/v2/nc_064_pg_minimal_dbs'; import * as nc_064_pg_minimal_dbs from '~/meta/migrations/v2/nc_064_pg_minimal_dbs';
import * as nc_065_encrypt_flag from '~/meta/migrations/v2/nc_065_encrypt_flag';
// Create a custom migration source class // Create a custom migration source class
export default class XcMigrationSourcev2 { export default class XcMigrationSourcev2 {
@ -113,6 +114,7 @@ export default class XcMigrationSourcev2 {
'nc_062_integration_store', 'nc_062_integration_store',
'nc_063_form_field_filter', 'nc_063_form_field_filter',
'nc_064_pg_minimal_dbs', 'nc_064_pg_minimal_dbs',
'nc_065_encrypt_flag',
]); ]);
} }
@ -228,6 +230,8 @@ export default class XcMigrationSourcev2 {
return nc_063_form_field_filter; return nc_063_form_field_filter;
case 'nc_064_pg_minimal_dbs': case 'nc_064_pg_minimal_dbs':
return nc_064_pg_minimal_dbs; return nc_064_pg_minimal_dbs;
case 'nc_065_encrypt_flag':
return nc_065_encrypt_flag;
} }
} }
} }

8
packages/nocodb/src/meta/migrations/v2/nc_037_rename_project_and_base.ts

@ -7,7 +7,7 @@ const logger = new Logger('nc_036_rename_project_and_base');
const up = async (knex: Knex) => { const up = async (knex: Knex) => {
logger.log('Renaming base table'); logger.log('Renaming base table');
if (await knex.schema.hasTable(MetaTableOldV2.BASES)) if (await knex.schema.hasTable(MetaTableOldV2.BASES))
await knex.schema.renameTable(MetaTableOldV2.BASES, MetaTable.BASES); await knex.schema.renameTable(MetaTableOldV2.BASES, MetaTable.SOURCES);
logger.log('Renaming `base_id` column to `source_id`'); logger.log('Renaming `base_id` column to `source_id`');
if (await knex.schema.hasColumn(MetaTable.MODELS, 'base_id')) if (await knex.schema.hasColumn(MetaTable.MODELS, 'base_id'))
@ -349,10 +349,10 @@ const up = async (knex: Knex) => {
}); });
logger.log( logger.log(
`Renaming 'project_id' column to 'base_id' in '${MetaTable.BASES}' table`, `Renaming 'project_id' column to 'base_id' in '${MetaTable.SOURCES}' table`,
); );
if (await knex.schema.hasColumn(MetaTable.BASES, 'project_id')) if (await knex.schema.hasColumn(MetaTable.SOURCES, 'project_id'))
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.renameColumn('project_id', 'base_id'); table.renameColumn('project_id', 'base_id');
}); });

4
packages/nocodb/src/meta/migrations/v2/nc_042_integrations.ts

@ -24,7 +24,7 @@ const up = async (knex: Knex) => {
table.timestamps(true, true); table.timestamps(true, true);
}); });
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.string('fk_integration_id', 20); table.string('fk_integration_id', 20);
}); });
@ -36,7 +36,7 @@ const up = async (knex: Knex) => {
const down = async (knex: Knex) => { const down = async (knex: Knex) => {
await knex.schema.dropTable(MetaTable.INTEGRATIONS); await knex.schema.dropTable(MetaTable.INTEGRATIONS);
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.dropColumn('fk_integration_id'); table.dropColumn('fk_integration_id');
}); });
}; };

4
packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts

@ -269,7 +269,7 @@ const up = async (knex: Knex) => {
// Drop existing base_id indexes // Drop existing base_id indexes
const dropBaseIdIndexes = [ const dropBaseIdIndexes = [
MetaTable.AUDIT, MetaTable.AUDIT,
MetaTable.BASES, MetaTable.SOURCES,
MetaTable.MODELS, MetaTable.MODELS,
MetaTable.PROJECT_USERS, MetaTable.PROJECT_USERS,
MetaTable.SYNC_SOURCE, MetaTable.SYNC_SOURCE,
@ -349,7 +349,7 @@ const up = async (knex: Knex) => {
MetaTable.MAP_VIEW, MetaTable.MAP_VIEW,
MetaTable.MODELS, MetaTable.MODELS,
MetaTable.SORT, MetaTable.SORT,
MetaTable.BASES, MetaTable.SOURCES,
MetaTable.SYNC_LOGS, MetaTable.SYNC_LOGS,
MetaTable.SYNC_SOURCE, MetaTable.SYNC_SOURCE,
MetaTable.VIEWS, MetaTable.VIEWS,

4
packages/nocodb/src/meta/migrations/v2/nc_051_source_readonly_columns.ts

@ -2,14 +2,14 @@ import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => { const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.boolean('is_schema_readonly').defaultTo(false); table.boolean('is_schema_readonly').defaultTo(false);
table.boolean('is_data_readonly').defaultTo(false); table.boolean('is_data_readonly').defaultTo(false);
}); });
}; };
const down = async (knex: Knex) => { const down = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.dropColumn('is_schema_readonly'); table.dropColumn('is_schema_readonly');
table.dropColumn('is_data_readonly'); table.dropColumn('is_data_readonly');
}); });

4
packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts

@ -107,7 +107,7 @@ export const replaceLongBaseIds = async (knex: Knex) => {
MetaTable.MAP_VIEW, MetaTable.MAP_VIEW,
MetaTable.MODELS, MetaTable.MODELS,
MetaTable.SORT, MetaTable.SORT,
MetaTable.BASES, MetaTable.SOURCES,
MetaTable.SYNC_LOGS, MetaTable.SYNC_LOGS,
MetaTable.SYNC_SOURCE, MetaTable.SYNC_SOURCE,
MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE, MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE,
@ -147,7 +147,7 @@ const tablesToAlterBaseId = [
MetaTable.MAP_VIEW_COLUMNS, MetaTable.MAP_VIEW_COLUMNS,
MetaTable.MODELS, MetaTable.MODELS,
MetaTable.SORT, MetaTable.SORT,
MetaTable.BASES, MetaTable.SOURCES,
MetaTable.SYNC_LOGS, MetaTable.SYNC_LOGS,
MetaTable.SYNC_SOURCE, MetaTable.SYNC_SOURCE,
MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE, MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE,

18
packages/nocodb/src/meta/migrations/v2/nc_056_integration.ts

@ -36,8 +36,8 @@ const up = async (knex: Knex) => {
}); });
} }
if (!(await knex.schema.hasColumn(MetaTable.BASES, 'fk_integration_id'))) { if (!(await knex.schema.hasColumn(MetaTable.SOURCES, 'fk_integration_id'))) {
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.string('fk_integration_id', 20).index(); table.string('fk_integration_id', 20).index();
}); });
} }
@ -45,18 +45,18 @@ const up = async (knex: Knex) => {
hrTime = process.hrtime(); hrTime = process.hrtime();
// get all external sources, add them to integrations table and map back to bases // get all external sources, add them to integrations table and map back to bases
const sources = await knex(MetaTable.BASES) const sources = await knex(MetaTable.SOURCES)
.select(`${MetaTable.BASES}.*`) .select(`${MetaTable.SOURCES}.*`)
.select(`${MetaTable.PROJECT_USERS}.fk_user_id as created_by`) .select(`${MetaTable.PROJECT_USERS}.fk_user_id as created_by`)
.innerJoin( .innerJoin(
MetaTable.PROJECT, MetaTable.PROJECT,
`${MetaTable.BASES}.base_id`, `${MetaTable.SOURCES}.base_id`,
`${MetaTable.PROJECT}.id`, `${MetaTable.PROJECT}.id`,
) )
.where((qb) => .where((qb) =>
qb qb
.where(`${MetaTable.BASES}.is_meta`, false) .where(`${MetaTable.SOURCES}.is_meta`, false)
.orWhereNull(`${MetaTable.BASES}.is_meta`), .orWhereNull(`${MetaTable.SOURCES}.is_meta`),
) )
.leftJoin(MetaTable.PROJECT_USERS, (qb) => { .leftJoin(MetaTable.PROJECT_USERS, (qb) => {
qb.on( qb.on(
@ -86,7 +86,7 @@ const up = async (knex: Knex) => {
}; };
await knex(MetaTable.INTEGRATIONS).insert(integration); await knex(MetaTable.INTEGRATIONS).insert(integration);
await knex(MetaTable.BASES).where('id', source.id).update({ await knex(MetaTable.SOURCES).where('id', source.id).update({
fk_integration_id: integrationId, fk_integration_id: integrationId,
}); });
} }
@ -97,7 +97,7 @@ const up = async (knex: Knex) => {
const down = async (knex: Knex) => { const down = async (knex: Knex) => {
await knex.schema.dropTable(MetaTable.INTEGRATIONS); await knex.schema.dropTable(MetaTable.INTEGRATIONS);
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.dropColumn('fk_integration_id'); table.dropColumn('fk_integration_id');
}); });
}; };

4
packages/nocodb/src/meta/migrations/v2/nc_064_pg_minimal_dbs.ts

@ -2,13 +2,13 @@ import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => { const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.boolean('is_local').defaultTo(false); table.boolean('is_local').defaultTo(false);
}); });
}; };
const down = async (knex: Knex) => { const down = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.BASES, (table) => { await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.dropColumn('is_local'); table.dropColumn('is_local');
}); });
}; };

22
packages/nocodb/src/meta/migrations/v2/nc_065_encrypt_flag.ts

@ -0,0 +1,22 @@
import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.boolean('is_encrypted').defaultTo(false);
});
await knex.schema.alterTable(MetaTable.INTEGRATIONS, (table) => {
table.boolean('is_encrypted').defaultTo(false);
});
};
const down = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.SOURCES, (table) => {
table.dropColumn('is_encrypted');
});
await knex.schema.alterTable(MetaTable.INTEGRATIONS, (table) => {
table.dropColumn('is_encrypted');
});
};
export { up, down };

257
packages/nocodb/src/models/Integration.spec.ts

@ -0,0 +1,257 @@
import { IntegrationsType } from 'nocodb-sdk';
import { Integration } from '~/models';
import { MetaTable } from '~/utils/globals';
import { decryptPropIfRequired, isEE } from '~/utils';
jest.mock('~/Noco');
describe('Integration Model', () => {
let integration: Integration;
let mockNcMeta: jest.Mocked<any>;
beforeEach(() => {
mockNcMeta = {
metaList: jest.fn(),
metaGet2: jest.fn(),
metaInsert2: jest.fn(),
metaUpdate: jest.fn(),
metaDelete: jest.fn(),
metaGetNextOrder: jest.fn(),
};
integration = new Integration({
id: 'test-id',
title: 'Test Integration',
base_id: 'project-1',
});
});
afterEach(() => {
jest.clearAllMocks();
});
describe('list', () => {
it('should list integrations', async () => {
const mockIntegrations = [
{ id: '1', title: 'Integration 1' },
{ id: '2', title: 'Integration 2' },
];
// Mock the knex function
mockNcMeta.knex = jest.fn().mockReturnValue({
select: jest.fn().mockReturnThis(),
from: jest.fn().mockReturnThis(),
where: jest.fn().mockReturnThis(),
whereNull: jest.fn().mockReturnThis(),
orWhereNull: jest.fn().mockReturnThis(),
leftJoin: jest.fn().mockReturnThis(),
andWhere: jest.fn().mockReturnThis(),
clone: jest.fn().mockReturnThis(),
limit: jest.fn().mockReturnThis(),
offset: jest.fn().mockReturnThis(),
orderBy: jest.fn().mockReturnThis(),
then: jest
.fn()
.mockImplementation((callback) =>
Promise.resolve(callback(mockIntegrations)),
),
});
const result = await Integration.list(
{
userId: 'user-id',
workspaceId: 'workspace-id',
},
mockNcMeta,
);
expect(result.list).toEqual(
mockIntegrations.map((i) => expect.objectContaining(i)),
);
// Verify that knex was called with the correct table
expect(mockNcMeta.knex).toHaveBeenCalledWith(MetaTable.INTEGRATIONS);
// Verify the chain of method calls
const knexMock = mockNcMeta.knex.mock.results[0].value;
expect(knexMock.where).toHaveBeenCalled();
expect(knexMock.orderBy).toHaveBeenCalledWith(
'nc_integrations_v2.order',
'asc',
);
});
});
describe('get', () => {
it('should get an integration by id', async () => {
const mockIntegration = { id: 'test-id', title: 'Test Integration' };
mockNcMeta.metaGet2.mockResolvedValue(mockIntegration);
const result = await Integration.get(
{
workspace_id: null,
},
'test-id',
false,
mockNcMeta,
);
expect(result).toBeInstanceOf(Integration);
expect(result).toEqual(expect.objectContaining(mockIntegration));
expect(mockNcMeta.metaGet2).toBeCalledWith(
null,
'workspace',
MetaTable.INTEGRATIONS,
isEE ? { fk_workspace_id: null, id: 'test-id' } : 'test-id',
null,
{ _or: [{ deleted: { neq: true } }, { deleted: { eq: null } }] },
);
});
});
describe('create', () => {
it('should create a new integration', async () => {
const newIntegration = {
id: 'new-id',
title: 'New Integration',
workspaceId: 'workspace-1',
config: {
client: 'pg',
},
};
mockNcMeta.metaInsert2.mockResolvedValue({
...newIntegration,
});
mockNcMeta.metaGet2.mockResolvedValue({
...newIntegration,
});
mockNcMeta.metaGetNextOrder.mockResolvedValue(2);
const result = await Integration.createIntegration(
newIntegration,
mockNcMeta,
);
expect(result).toBeInstanceOf(Integration);
expect(result).toEqual(
expect.objectContaining({ id: 'new-id', ...newIntegration }),
);
expect(mockNcMeta.metaInsert2).toHaveBeenCalledWith(
'workspace-1',
'workspace',
MetaTable.INTEGRATIONS,
{
...newIntegration,
order: 2,
fk_workspace_id: 'workspace-1',
workspaceId: undefined,
id: undefined,
config: JSON.stringify(newIntegration.config),
is_encrypted: false,
},
);
});
});
describe('create with encryption', () => {
beforeAll(() => {
process.env.NC_CONNECTION_ENCRYPT_KEY = 'test-secret';
});
afterAll(() => {
process.env.NC_CONNECTION_ENCRYPT_KEY = undefined;
});
it('should create a new integration with encrypted config', async () => {
const newIntegration = {
id: 'new-id',
title: 'New Integration',
workspaceId: 'workspace-1',
config: {
client: 'pg',
},
};
mockNcMeta.metaInsert2.mockResolvedValue({
...newIntegration,
});
mockNcMeta.metaInsert2.mockResolvedValue({
...newIntegration,
});
mockNcMeta.metaGet2.mockResolvedValue({
...newIntegration,
});
mockNcMeta.metaGetNextOrder.mockResolvedValue(2);
const result = await Integration.createIntegration(
newIntegration,
mockNcMeta,
);
expect(result).toBeInstanceOf(Integration);
expect(result).toEqual(
expect.objectContaining({ id: 'new-id', ...newIntegration }),
);
// Extract the arguments used in the call
const calledWithArgs = mockNcMeta.metaInsert2.mock.calls[0][3];
// veify the 'config' field is encrypted
expect(calledWithArgs.config).not.toEqual(
JSON.stringify(newIntegration.config),
);
// Decrypt the 'config' field
const decryptedConfig = decryptPropIfRequired({ data: calledWithArgs });
// Verify the decrypted config matches the original integration
expect(decryptedConfig).toEqual(newIntegration.config);
});
});
describe('update', () => {
it('should update an existing integration', async () => {
const updateData = {
title: 'Updated Integration',
type: IntegrationsType.Database,
};
mockNcMeta.metaUpdate.mockResolvedValue({
id: 'test-id',
type: IntegrationsType.Database,
...updateData,
});
mockNcMeta.metaGet2.mockResolvedValue({
id: 'test-id',
type: IntegrationsType.Database,
...updateData,
});
await Integration.updateIntegration(
{
workspace_id: null,
},
'test-id',
updateData,
mockNcMeta,
);
expect(mockNcMeta.metaUpdate).toHaveBeenCalledWith(
null,
'workspace',
MetaTable.INTEGRATIONS,
updateData,
integration.id,
);
});
});
describe('delete', () => {
it('should delete an integration', async () => {
await integration.delete(mockNcMeta);
expect(mockNcMeta.metaDelete).toHaveBeenCalledWith(
undefined,
'workspace',
MetaTable.INTEGRATIONS,
integration.id,
);
});
});
});

64
packages/nocodb/src/models/Integration.ts

@ -1,4 +1,3 @@
import CryptoJS from 'crypto-js';
import type { IntegrationsType, SourceType } from 'nocodb-sdk'; import type { IntegrationsType, SourceType } from 'nocodb-sdk';
import type { BoolType, IntegrationType } from 'nocodb-sdk'; import type { BoolType, IntegrationType } from 'nocodb-sdk';
import type { NcContext } from '~/interface/config'; import type { NcContext } from '~/interface/config';
@ -11,7 +10,12 @@ import {
prepareForDb, prepareForDb,
stringifyMetaProp, stringifyMetaProp,
} from '~/utils/modelUtils'; } from '~/utils/modelUtils';
import { partialExtract } from '~/utils'; import {
decryptPropIfRequired,
encryptPropIfRequired,
isEncryptionRequired,
partialExtract,
} from '~/utils';
import { PagedResponseImpl } from '~/helpers/PagedResponse'; import { PagedResponseImpl } from '~/helpers/PagedResponse';
export default class Integration implements IntegrationType { export default class Integration implements IntegrationType {
@ -27,6 +31,7 @@ export default class Integration implements IntegrationType {
meta?: any; meta?: any;
created_by?: string; created_by?: string;
sources?: Partial<SourceType>[]; sources?: Partial<SourceType>[];
is_encrypted?: BoolType;
constructor(integration: Partial<IntegrationType>) { constructor(integration: Partial<IntegrationType>) {
Object.assign(this, integration); Object.assign(this, integration);
@ -36,12 +41,18 @@ export default class Integration implements IntegrationType {
return integration && new Integration(integration); return integration && new Integration(integration);
} }
protected static encryptConfigIfRequired(obj: Record<string, unknown>) {
obj.config = encryptPropIfRequired({ data: obj });
obj.is_encrypted = isEncryptionRequired();
}
public static async createIntegration( public static async createIntegration(
integration: IntegrationType & { integration: IntegrationType & {
workspaceId?: string; workspaceId?: string;
created_at?; created_at?;
updated_at?; updated_at?;
meta?: any; meta?: any;
is_encrypted?: BoolType;
}, },
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
@ -54,12 +65,10 @@ export default class Integration implements IntegrationType {
'meta', 'meta',
'created_by', 'created_by',
'is_private', 'is_private',
'is_encrypted',
]); ]);
insertObj.config = CryptoJS.AES.encrypt( this.encryptConfigIfRequired(insertObj);
JSON.stringify(insertObj.config),
Noco.getConfig()?.auth?.jwt?.secret,
).toString();
if ('meta' in insertObj) { if ('meta' in insertObj) {
insertObj.meta = stringifyMetaProp(insertObj); insertObj.meta = stringifyMetaProp(insertObj);
@ -99,6 +108,7 @@ export default class Integration implements IntegrationType {
integration: IntegrationType & { integration: IntegrationType & {
meta?: any; meta?: any;
deleted?: boolean; deleted?: boolean;
is_encrypted?: boolean;
}, },
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
@ -121,13 +131,16 @@ export default class Integration implements IntegrationType {
'deleted', 'deleted',
'config', 'config',
'is_private', 'is_private',
'is_encrypted',
]); ]);
if (updateObj.config) { if (updateObj.config) {
updateObj.config = CryptoJS.AES.encrypt( updateObj.config = encryptPropIfRequired({
JSON.stringify(integration.config), data: updateObj,
Noco.getConfig()?.auth?.jwt?.secret, });
).toString(); updateObj.is_encrypted = isEncryptionRequired();
this.encryptConfigIfRequired(updateObj);
} }
// type property is undefined even if not provided // type property is undefined even if not provided
@ -221,12 +234,12 @@ export default class Integration implements IntegrationType {
listQb listQb
.select( .select(
`${MetaTable.INTEGRATIONS}.*`, `${MetaTable.INTEGRATIONS}.*`,
ncMeta.knex.raw(`count(${MetaTable.BASES}.id) as source_count`), ncMeta.knex.raw(`count(${MetaTable.SOURCES}.id) as source_count`),
) )
.leftJoin( .leftJoin(
MetaTable.BASES, MetaTable.SOURCES,
`${MetaTable.INTEGRATIONS}.id`, `${MetaTable.INTEGRATIONS}.id`,
`${MetaTable.BASES}.fk_integration_id`, `${MetaTable.SOURCES}.fk_integration_id`,
) )
.groupBy(`${MetaTable.INTEGRATIONS}.id`); .groupBy(`${MetaTable.INTEGRATIONS}.id`);
} }
@ -335,12 +348,9 @@ export default class Integration implements IntegrationType {
} }
public getConfig(): any { public getConfig(): any {
const config = JSON.parse( const config = decryptPropIfRequired({
CryptoJS.AES.decrypt( data: this,
this.config, });
Noco.getConfig()?.auth?.jwt?.secret,
).toString(CryptoJS.enc.Utf8),
);
return config; return config;
} }
@ -369,23 +379,23 @@ export default class Integration implements IntegrationType {
} }
async getSources(ncMeta = Noco.ncMeta): Promise<any> { async getSources(ncMeta = Noco.ncMeta): Promise<any> {
const qb = ncMeta.knex(MetaTable.BASES); const qb = ncMeta.knex(MetaTable.SOURCES);
const sources = await qb const sources = await qb
.select(`${MetaTable.BASES}.id`) .select(`${MetaTable.SOURCES}.id`)
.select(`${MetaTable.BASES}.alias`) .select(`${MetaTable.SOURCES}.alias`)
.select(`${MetaTable.PROJECT}.title as project_title`) .select(`${MetaTable.PROJECT}.title as project_title`)
.select(`${MetaTable.BASES}.base_id`) .select(`${MetaTable.SOURCES}.base_id`)
.innerJoin( .innerJoin(
MetaTable.PROJECT, MetaTable.PROJECT,
`${MetaTable.BASES}.base_id`, `${MetaTable.SOURCES}.base_id`,
`${MetaTable.PROJECT}.id`, `${MetaTable.PROJECT}.id`,
) )
.where(`${MetaTable.BASES}.fk_integration_id`, this.id) .where(`${MetaTable.SOURCES}.fk_integration_id`, this.id)
.where((whereQb) => { .where((whereQb) => {
whereQb whereQb
.where(`${MetaTable.BASES}.deleted`, false) .where(`${MetaTable.SOURCES}.deleted`, false)
.orWhereNull(`${MetaTable.BASES}.deleted`); .orWhereNull(`${MetaTable.SOURCES}.deleted`);
}) })
.where((whereQb) => { .where((whereQb) => {
whereQb whereQb

114
packages/nocodb/src/models/Source.ts

@ -1,5 +1,4 @@
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import CryptoJS from 'crypto-js';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import type { DriverClient } from '~/utils/nc-config'; import type { DriverClient } from '~/utils/nc-config';
import type { BoolType, SourceType } from 'nocodb-sdk'; import type { BoolType, SourceType } from 'nocodb-sdk';
@ -24,8 +23,14 @@ import {
} from '~/utils/modelUtils'; } from '~/utils/modelUtils';
import { JobsRedis } from '~/modules/jobs/redis/jobs-redis'; import { JobsRedis } from '~/modules/jobs/redis/jobs-redis';
import { InstanceCommands } from '~/interface/Jobs'; import { InstanceCommands } from '~/interface/Jobs';
import { deepMerge, partialExtract } from '~/utils';
import View from '~/models/View'; import View from '~/models/View';
import {
decryptPropIfRequired,
deepMerge,
encryptPropIfRequired,
isEncryptionRequired,
partialExtract,
} from '~/utils';
export default class Source implements SourceType { export default class Source implements SourceType {
id?: string; id?: string;
@ -47,6 +52,7 @@ export default class Source implements SourceType {
fk_integration_id?: string; fk_integration_id?: string;
integration_config?: string; integration_config?: string;
integration_title?: string; integration_title?: string;
is_encrypted?: boolean;
constructor(source: Partial<SourceType>) { constructor(source: Partial<SourceType>) {
Object.assign(this, source); Object.assign(this, source);
@ -56,6 +62,11 @@ export default class Source implements SourceType {
return source && new Source(source); return source && new Source(source);
} }
protected static encryptConfigIfRequired(obj: Record<string, unknown>) {
obj.config = encryptPropIfRequired({ data: obj });
obj.is_encrypted = isEncryptionRequired();
}
public static async createBase( public static async createBase(
context: NcContext, context: NcContext,
source: SourceType & { source: SourceType & {
@ -63,6 +74,7 @@ export default class Source implements SourceType {
created_at?; created_at?;
updated_at?; updated_at?;
meta?: any; meta?: any;
is_encrypted?: boolean;
}, },
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
@ -81,34 +93,32 @@ export default class Source implements SourceType {
'is_schema_readonly', 'is_schema_readonly',
'is_data_readonly', 'is_data_readonly',
'fk_integration_id', 'fk_integration_id',
'is_encrypted',
]); ]);
insertObj.config = CryptoJS.AES.encrypt( this.encryptConfigIfRequired(insertObj);
JSON.stringify(source.config),
Noco.getConfig()?.auth?.jwt?.secret,
).toString();
if ('meta' in insertObj) { if ('meta' in insertObj) {
insertObj.meta = stringifyMetaProp(insertObj); insertObj.meta = stringifyMetaProp(insertObj);
} }
insertObj.order = await ncMeta.metaGetNextOrder(MetaTable.BASES, { insertObj.order = await ncMeta.metaGetNextOrder(MetaTable.SOURCES, {
base_id: source.baseId, base_id: source.baseId,
}); });
const { id } = await ncMeta.metaInsert2( const { id } = await ncMeta.metaInsert2(
context.workspace_id, context.workspace_id,
context.base_id, context.base_id,
MetaTable.BASES, MetaTable.SOURCES,
insertObj, insertObj,
); );
const returnBase = await this.get(context, id, false, ncMeta); const returnBase = await this.get(context, id, false, ncMeta);
await NocoCache.appendToList( await NocoCache.appendToList(
CacheScope.BASE, CacheScope.SOURCE,
[source.baseId], [source.baseId],
`${CacheScope.BASE}:${id}`, `${CacheScope.SOURCE}:${id}`,
); );
return returnBase; return returnBase;
@ -121,6 +131,7 @@ export default class Source implements SourceType {
meta?: any; meta?: any;
deleted?: boolean; deleted?: boolean;
fk_sql_executor_id?: string; fk_sql_executor_id?: string;
is_encrypted?: boolean;
}, },
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
@ -144,13 +155,11 @@ export default class Source implements SourceType {
'is_schema_readonly', 'is_schema_readonly',
'is_data_readonly', 'is_data_readonly',
'fk_integration_id', 'fk_integration_id',
'is_encrypted',
]); ]);
if (updateObj.config) { if (updateObj.config) {
updateObj.config = CryptoJS.AES.encrypt( this.encryptConfigIfRequired(updateObj);
JSON.stringify(source.config),
Noco.getConfig()?.auth?.jwt?.secret,
).toString();
} }
// type property is undefined even if not provided // type property is undefined even if not provided
@ -164,7 +173,7 @@ export default class Source implements SourceType {
// if order is missing (possible in old versions), get next order // if order is missing (possible in old versions), get next order
if (!oldSource.order && !updateObj.order) { if (!oldSource.order && !updateObj.order) {
updateObj.order = await ncMeta.metaGetNextOrder(MetaTable.BASES, { updateObj.order = await ncMeta.metaGetNextOrder(MetaTable.SOURCES, {
base_id: oldSource.base_id, base_id: oldSource.base_id,
}); });
@ -186,7 +195,7 @@ export default class Source implements SourceType {
// if order is 1 for non-default source, move it to last // if order is 1 for non-default source, move it to last
if (oldSource.order <= 1 && !updateObj.order) { if (oldSource.order <= 1 && !updateObj.order) {
updateObj.order = await ncMeta.metaGetNextOrder(MetaTable.BASES, { updateObj.order = await ncMeta.metaGetNextOrder(MetaTable.SOURCES, {
base_id: oldSource.base_id, base_id: oldSource.base_id,
}); });
} }
@ -195,13 +204,13 @@ export default class Source implements SourceType {
await ncMeta.metaUpdate( await ncMeta.metaUpdate(
context.workspace_id, context.workspace_id,
context.base_id, context.base_id,
MetaTable.BASES, MetaTable.SOURCES,
prepareForDb(updateObj), prepareForDb(updateObj),
oldSource.id, oldSource.id,
); );
await NocoCache.update( await NocoCache.update(
`${CacheScope.BASE}:${sourceId}`, `${CacheScope.SOURCE}:${sourceId}`,
prepareForResponse(updateObj), prepareForResponse(updateObj),
); );
@ -223,20 +232,22 @@ export default class Source implements SourceType {
args: { baseId: string }, args: { baseId: string },
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
): Promise<Source[]> { ): Promise<Source[]> {
const cachedList = await NocoCache.getList(CacheScope.BASE, [args.baseId]); const cachedList = await NocoCache.getList(CacheScope.SOURCE, [
args.baseId,
]);
let { list: sourceDataList } = cachedList; let { list: sourceDataList } = cachedList;
const { isNoneList } = cachedList; const { isNoneList } = cachedList;
if (!isNoneList && !sourceDataList.length) { if (!isNoneList && !sourceDataList.length) {
const qb = ncMeta const qb = ncMeta
.knex(MetaTable.BASES) .knex(MetaTable.SOURCES)
.select(`${MetaTable.BASES}.*`) .select(`${MetaTable.SOURCES}.*`)
.where(`${MetaTable.BASES}.base_id`, context.base_id) .where(`${MetaTable.SOURCES}.base_id`, context.base_id)
.where((whereQb) => { .where((whereQb) => {
whereQb whereQb
.where(`${MetaTable.BASES}.deleted`, false) .where(`${MetaTable.SOURCES}.deleted`, false)
.orWhereNull(`${MetaTable.BASES}.deleted`); .orWhereNull(`${MetaTable.SOURCES}.deleted`);
}) })
.orderBy(`${MetaTable.BASES}.order`, 'asc'); .orderBy(`${MetaTable.SOURCES}.order`, 'asc');
this.extendQb(qb, context); this.extendQb(qb, context);
@ -247,7 +258,7 @@ export default class Source implements SourceType {
source.meta = parseMetaProp(source, 'meta'); source.meta = parseMetaProp(source, 'meta');
} }
await NocoCache.setList(CacheScope.BASE, [args.baseId], sourceDataList); await NocoCache.setList(CacheScope.SOURCE, [args.baseId], sourceDataList);
} }
sourceDataList.sort( sourceDataList.sort(
@ -268,23 +279,23 @@ export default class Source implements SourceType {
let sourceData = let sourceData =
id && id &&
(await NocoCache.get( (await NocoCache.get(
`${CacheScope.BASE}:${id}`, `${CacheScope.SOURCE}:${id}`,
CacheGetType.TYPE_OBJECT, CacheGetType.TYPE_OBJECT,
)); ));
if (!sourceData) { if (!sourceData) {
const qb = ncMeta const qb = ncMeta
.knex(MetaTable.BASES) .knex(MetaTable.SOURCES)
.select(`${MetaTable.BASES}.*`) .select(`${MetaTable.SOURCES}.*`)
.where(`${MetaTable.BASES}.id`, id) .where(`${MetaTable.SOURCES}.id`, id)
.where(`${MetaTable.BASES}.base_id`, context.base_id); .where(`${MetaTable.SOURCES}.base_id`, context.base_id);
this.extendQb(qb, context); this.extendQb(qb, context);
if (!force) { if (!force) {
qb.where((whereQb) => { qb.where((whereQb) => {
whereQb whereQb
.where(`${MetaTable.BASES}.deleted`, false) .where(`${MetaTable.SOURCES}.deleted`, false)
.orWhereNull(`${MetaTable.BASES}.deleted`); .orWhereNull(`${MetaTable.SOURCES}.deleted`);
}); });
} }
@ -294,7 +305,7 @@ export default class Source implements SourceType {
sourceData.meta = parseMetaProp(sourceData, 'meta'); sourceData.meta = parseMetaProp(sourceData, 'meta');
} }
await NocoCache.set(`${CacheScope.BASE}:${id}`, sourceData); await NocoCache.set(`${CacheScope.SOURCE}:${id}`, sourceData);
} }
return this.castType(sourceData); return this.castType(sourceData);
} }
@ -329,12 +340,9 @@ export default class Source implements SourceType {
return config; return config;
} }
const config = JSON.parse( const config = decryptPropIfRequired({
CryptoJS.AES.decrypt( data: this,
this.config, });
Noco.getConfig()?.auth?.jwt?.secret,
).toString(CryptoJS.enc.Utf8),
);
if (skipIntegrationConfig) { if (skipIntegrationConfig) {
return config; return config;
@ -344,12 +352,10 @@ export default class Source implements SourceType {
return config; return config;
} }
const integrationConfig = JSON.parse( const integrationConfig = decryptPropIfRequired({
CryptoJS.AES.decrypt( data: this,
this.integration_config, prop: 'integration_config',
Noco.getConfig()?.auth?.jwt?.secret, });
).toString(CryptoJS.enc.Utf8),
);
// merge integration config with source config // merge integration config with source config
// override integration config with source config if exists // override integration config with source config if exists
// only override database and searchPath // only override database and searchPath
@ -474,12 +480,12 @@ export default class Source implements SourceType {
const res = await ncMeta.metaDelete( const res = await ncMeta.metaDelete(
context.workspace_id, context.workspace_id,
context.base_id, context.base_id,
MetaTable.BASES, MetaTable.SOURCES,
this.id, this.id,
); );
await NocoCache.deepDel( await NocoCache.deepDel(
`${CacheScope.BASE}:${this.id}`, `${CacheScope.SOURCE}:${this.id}`,
CacheDelDirection.CHILD_TO_PARENT, CacheDelDirection.CHILD_TO_PARENT,
); );
@ -504,7 +510,7 @@ export default class Source implements SourceType {
await Source.update(context, this.id, { deleted: true }, ncMeta); await Source.update(context, this.id, { deleted: true }, ncMeta);
await NocoCache.deepDel( await NocoCache.deepDel(
`${CacheScope.BASE}:${this.id}`, `${CacheScope.SOURCE}:${this.id}`,
CacheDelDirection.CHILD_TO_PARENT, CacheDelDirection.CHILD_TO_PARENT,
); );
} }
@ -526,14 +532,14 @@ export default class Source implements SourceType {
await ncMeta.metaUpdate( await ncMeta.metaUpdate(
context.workspace_id, context.workspace_id,
context.base_id, context.base_id,
MetaTable.BASES, MetaTable.SOURCES,
{ {
erd_uuid: this.erd_uuid, erd_uuid: this.erd_uuid,
}, },
this.id, this.id,
); );
await NocoCache.update(`${CacheScope.BASE}:${this.id}`, { await NocoCache.update(`${CacheScope.SOURCE}:${this.id}`, {
erd_uuid: this.erd_uuid, erd_uuid: this.erd_uuid,
}); });
} }
@ -548,14 +554,14 @@ export default class Source implements SourceType {
await ncMeta.metaUpdate( await ncMeta.metaUpdate(
context.workspace_id, context.workspace_id,
context.base_id, context.base_id,
MetaTable.BASES, MetaTable.SOURCES,
{ {
erd_uuid: this.erd_uuid, erd_uuid: this.erd_uuid,
}, },
this.id, this.id,
); );
await NocoCache.update(`${CacheScope.BASE}:${this.id}`, { await NocoCache.update(`${CacheScope.SOURCE}:${this.id}`, {
erd_uuid: this.erd_uuid, erd_uuid: this.erd_uuid,
}); });
} }
@ -579,7 +585,7 @@ export default class Source implements SourceType {
`${MetaTable.INTEGRATIONS}.title as integration_title`, `${MetaTable.INTEGRATIONS}.title as integration_title`,
).leftJoin( ).leftJoin(
MetaTable.INTEGRATIONS, MetaTable.INTEGRATIONS,
`${MetaTable.BASES}.fk_integration_id`, `${MetaTable.SOURCES}.fk_integration_id`,
`${MetaTable.INTEGRATIONS}.id`, `${MetaTable.INTEGRATIONS}.id`,
); );
} }

1
packages/nocodb/src/models/UserRefreshToken.ts

@ -25,7 +25,6 @@ export default class UserRefreshToken {
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
// clear old invalid tokens before inserting new one // clear old invalid tokens before inserting new one
// todo: verify the populated sql query
await ncMeta.metaDelete( await ncMeta.metaDelete(
RootScopes.ROOT, RootScopes.ROOT,
RootScopes.ROOT, RootScopes.ROOT,

6
packages/nocodb/src/providers/init-meta-service.provider.ts

@ -14,6 +14,7 @@ import { NcConfig, prepareEnv } from '~/utils/nc-config';
import { MetaTable, RootScopes } from '~/utils/globals'; import { MetaTable, RootScopes } from '~/utils/globals';
import { updateMigrationJobsState } from '~/helpers/migrationJobs'; import { updateMigrationJobsState } from '~/helpers/migrationJobs';
import { initBaseBehavior } from '~/helpers/initBaseBehaviour'; import { initBaseBehavior } from '~/helpers/initBaseBehaviour';
import initDataSourceEncryption from '~/helpers/initDataSourceEncryption';
export const InitMetaServiceProvider: FactoryProvider = { export const InitMetaServiceProvider: FactoryProvider = {
// initialize app, // initialize app,
@ -30,7 +31,7 @@ export const InitMetaServiceProvider: FactoryProvider = {
const config = await NcConfig.createByEnv(); const config = await NcConfig.createByEnv();
// set version // set version
process.env.NC_VERSION = '0111005'; process.env.NC_VERSION = '0225002';
// set migration jobs version // set migration jobs version
process.env.NC_MIGRATION_JOBS_VERSION = '2'; process.env.NC_MIGRATION_JOBS_VERSION = '2';
@ -116,6 +117,9 @@ export const InitMetaServiceProvider: FactoryProvider = {
// decide base behavior based on env and database permissions // decide base behavior based on env and database permissions
await initBaseBehavior(); await initBaseBehavior();
// encrypt datasource if secret is set
await initDataSourceEncryption(metaService);
return metaService; return metaService;
}, },
provide: MetaService, provide: MetaService,

16
packages/nocodb/src/services/base-users/base-users.service.spec.ts

@ -1,13 +1,25 @@
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { BaseUsersService } from './base-users.service'; import { mock } from 'jest-mock-extended';
import type { TestingModule } from '@nestjs/testing'; import type { TestingModule } from '@nestjs/testing';
import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface';
import { BaseUsersService } from '~/services/base-users/base-users.service';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
describe('BaseUsersService', () => { describe('BaseUsersService', () => {
let service: BaseUsersService; let service: BaseUsersService;
beforeEach(async () => { beforeEach(async () => {
const eventEmitter = mock<IEventEmitter>();
const module: TestingModule = await Test.createTestingModule({ const module: TestingModule = await Test.createTestingModule({
providers: [BaseUsersService], providers: [
BaseUsersService,
AppHooksService,
{
provide: 'IEventEmitter',
useValue: eventEmitter,
},
],
}).compile(); }).compile();
service = module.get<BaseUsersService>(BaseUsersService); service = module.get<BaseUsersService>(BaseUsersService);

6
packages/nocodb/src/services/integrations.service.ts

@ -120,7 +120,7 @@ export class IntegrationsService {
// get linked sources // get linked sources
const sourceListQb = ncMeta const sourceListQb = ncMeta
.knex(MetaTable.BASES) .knex(MetaTable.SOURCES)
.where({ .where({
fk_integration_id: integration.id, fk_integration_id: integration.id,
}) })
@ -309,7 +309,7 @@ export class IntegrationsService {
const sources = await ncMeta.metaList2( const sources = await ncMeta.metaList2(
integration.fk_workspace_id, integration.fk_workspace_id,
RootScopes.WORKSPACE, RootScopes.WORKSPACE,
MetaTable.BASES, MetaTable.SOURCES,
{ {
condition: { condition: {
fk_integration_id: integration.id, fk_integration_id: integration.id,
@ -336,7 +336,7 @@ export class IntegrationsService {
const source = new Source(sourceObj); const source = new Source(sourceObj);
// update the cache with the new config(encrypted) // update the cache with the new config(encrypted)
await NocoCache.update(`${CacheScope.BASE}:${source.id}`, { await NocoCache.update(`${CacheScope.SOURCE}:${source.id}`, {
integration_config: integration.config, integration_config: integration.config,
}); });

15
packages/nocodb/src/services/org-users.service.spec.ts

@ -1,13 +1,24 @@
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { OrgUsersService } from './org-users.service'; import { mock } from 'jest-mock-extended';
import type { TestingModule } from '@nestjs/testing'; import type { TestingModule } from '@nestjs/testing';
import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface';
import { OrgUsersService } from '~/services/org-users.service';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
describe('OrgUsersService', () => { describe('OrgUsersService', () => {
let service: OrgUsersService; let service: OrgUsersService;
beforeEach(async () => { beforeEach(async () => {
const eventEmitter = mock<IEventEmitter>();
const module: TestingModule = await Test.createTestingModule({ const module: TestingModule = await Test.createTestingModule({
providers: [OrgUsersService], providers: [
OrgUsersService,
AppHooksService,
{
provide: 'IEventEmitter',
useValue: eventEmitter,
},
],
}).compile(); }).compile();
service = module.get<OrgUsersService>(OrgUsersService); service = module.get<OrgUsersService>(OrgUsersService);

0
packages/nocodb/src/services/users/index.ts

35
packages/nocodb/src/services/users/users.service.spec.ts

@ -1,13 +1,44 @@
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { UsersService } from './users.service'; import { mock } from 'jest-mock-extended';
import type { TestingModule } from '@nestjs/testing'; import type { TestingModule } from '@nestjs/testing';
import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface';
import type { MetaService } from '~/meta/meta.service';
import { BasesService } from '~/services/bases.service';
import { UsersService } from '~/services/users/users.service';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { WorkspacesService } from '~/ee/services/workspaces.service';
import { TablesService } from '~/services/tables.service';
import { ColumnsService } from '~/services/columns.service';
import { MetaDiffsService } from '~/services/meta-diffs.service';
describe('UsersService', () => { describe('UsersService', () => {
let service: UsersService; let service: UsersService;
beforeEach(async () => { beforeEach(async () => {
const eventEmitter = mock<IEventEmitter>();
const metaService = mock<MetaService>();
const module: TestingModule = await Test.createTestingModule({ const module: TestingModule = await Test.createTestingModule({
providers: [UsersService], providers: [
AppHooksService,
WorkspacesService,
BasesService,
TablesService,
ColumnsService,
MetaDiffsService,
{
provide: 'IEventEmitter',
useValue: eventEmitter,
},
{
provide: 'MetaService',
useValue: metaService,
},
{
provide: 'JobsService',
useValue: {},
},
UsersService,
],
}).compile(); }).compile();
service = module.get<UsersService>(UsersService); service = module.get<UsersService>(UsersService);

54
packages/nocodb/src/utils/encryptDecrypt.ts

@ -0,0 +1,54 @@
import CryptoJS from 'crypto-js';
export const getCredentialEncryptSecret = () =>
process.env.NC_CONNECTION_ENCRYPT_KEY;
export const isEncryptionRequired = (secret = getCredentialEncryptSecret()) => {
return !!secret;
};
export const encryptPropIfRequired = ({
data,
prop = 'config',
secret = getCredentialEncryptSecret(),
}: {
data: Record<string, any>;
prop?: string;
secret?: string;
}) => {
if (!data || data[prop] === null || data[prop] === undefined) {
return;
}
if (!secret) {
return JSON.stringify(data[prop]);
}
return CryptoJS.AES.encrypt(JSON.stringify(data[prop]), secret).toString();
};
export const decryptPropIfRequired = ({
data,
prop = 'config',
secret = getCredentialEncryptSecret(),
}: {
data: Record<string, any>;
prop?: string;
secret?: string;
}) => {
if (!data || data[prop] === null || data[prop] === undefined) {
return;
}
let jsonString = data[prop];
if (secret) {
try {
jsonString = CryptoJS.AES.decrypt(data[prop], secret).toString(
CryptoJS.enc.Utf8,
);
} catch {
throw new Error('Config decryption failed');
}
}
return typeof jsonString === 'string' ? JSON.parse(jsonString) : jsonString;
};

6
packages/nocodb/src/utils/globals.ts

@ -1,6 +1,6 @@
export enum MetaTable { export enum MetaTable {
PROJECT = 'nc_bases_v2', PROJECT = 'nc_bases_v2',
BASES = 'nc_sources_v2', SOURCES = 'nc_source_v2',
MODELS = 'nc_models_v2', MODELS = 'nc_models_v2',
COLUMNS = 'nc_columns_v2', COLUMNS = 'nc_columns_v2',
COLUMN_VALIDATIONS = 'nc_columns_validations_v2', COLUMN_VALIDATIONS = 'nc_columns_validations_v2',
@ -100,7 +100,7 @@ export const orderedMetaTables = [
MetaTable.COLUMN_VALIDATIONS, MetaTable.COLUMN_VALIDATIONS,
MetaTable.COLUMNS, MetaTable.COLUMNS,
MetaTable.MODELS, MetaTable.MODELS,
MetaTable.BASES, MetaTable.SOURCES,
MetaTable.PROJECT, MetaTable.PROJECT,
]; ];
@ -132,7 +132,7 @@ export const sakilaTableNames = [
export enum CacheScope { export enum CacheScope {
PROJECT = 'base', PROJECT = 'base',
BASE = 'source', SOURCE = 'source',
MODEL = 'model', MODEL = 'model',
COLUMN = 'column', COLUMN = 'column',
COL_PROP = 'colProp', COL_PROP = 'colProp',

1
packages/nocodb/src/utils/index.ts

@ -5,5 +5,6 @@ export * from './circularReplacer';
export * from './nocoExecute'; export * from './nocoExecute';
export { Tele as T } from './tele'; export { Tele as T } from './tele';
export * from './packageVersion'; export * from './packageVersion';
export * from './encryptDecrypt';
export const isEE = false; export const isEE = false;

19
packages/nocodb/src/utils/nc-config/helpers.ts

@ -13,17 +13,16 @@ import {
import { DriverClient } from './interfaces'; import { DriverClient } from './interfaces';
import type { Connection, DbConfig } from './interfaces'; import type { Connection, DbConfig } from './interfaces';
export async function prepareEnv() { export async function prepareEnv({
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { databaseUrlFile = process.env.NC_DATABASE_URL_FILE ||
const database_url = await promisify(fs.readFile)( process.env.DATABASE_URL_FILE,
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, databaseUrl = process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
'utf-8', } = {}) {
); if (databaseUrlFile) {
const database_url = await promisify(fs.readFile)(databaseUrlFile, 'utf-8');
process.env.NC_DB = jdbcToXcUrl(database_url); process.env.NC_DB = jdbcToXcUrl(database_url);
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { } else if (databaseUrl) {
process.env.NC_DB = jdbcToXcUrl( process.env.NC_DB = jdbcToXcUrl(databaseUrl);
process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
);
} }
} }

24
packages/nocodb/src/version-upgrader/NcUpgrader.ts

@ -1,16 +1,17 @@
import debug from 'debug'; import debug from 'debug';
import boxen from 'boxen'; import boxen from 'boxen';
import ncAttachmentUpgrader from './ncAttachmentUpgrader'; import ncAttachmentUpgrader from './upgraders/0101002_ncAttachmentUpgrader';
import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002'; import ncAttachmentUpgrader_0104002 from './upgraders/0104002_ncAttachmentUpgrader';
import ncStickyColumnUpgrader from './ncStickyColumnUpgrader'; import ncStickyColumnUpgrader from './upgraders/0105002_ncStickyColumnUpgrader';
import ncFilterUpgrader_0104004 from './ncFilterUpgrader_0104004'; import ncFilterUpgrader_0104004 from './upgraders/0104004_ncFilterUpgrader';
import ncFilterUpgrader_0105003 from './ncFilterUpgrader_0105003'; import ncFilterUpgrader_0105003 from './upgraders/0105003_ncFilterUpgrader';
import ncFilterUpgrader from './ncFilterUpgrader'; import ncFilterUpgrader from './upgraders/0100002_ncFilterUpgrader';
import ncHookUpgrader from './ncHookUpgrader'; import ncHookUpgrader from './upgraders/0105004_ncHookUpgrader';
import ncProjectConfigUpgrader from './ncProjectConfigUpgrader'; import ncProjectConfigUpgrader from './upgraders/0107004_ncProjectConfigUpgrader';
import ncXcdbLTARUpgrader from './ncXcdbLTARUpgrader'; import ncXcdbLTARUpgrader from './upgraders/0108002_ncXcdbLTARUpgrader';
import ncXcdbLTARIndexUpgrader from './ncXcdbLTARIndexUpgrader'; import ncXcdbLTARIndexUpgrader from './upgraders/0111002_ncXcdbLTARIndexUpgrader';
import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './ncXcdbCreatedAndUpdatedSystemFieldsUpgrader'; import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './upgraders/0111005_ncXcdbCreatedAndUpdatedSystemFieldsUpgrader';
import ncDatasourceDecrypt from './upgraders/0225002_ncDatasourceDecrypt';
import type { MetaService } from '~/meta/meta.service'; import type { MetaService } from '~/meta/meta.service';
import type { NcConfig } from '~/interface/config'; import type { NcConfig } from '~/interface/config';
import { T } from '~/utils'; import { T } from '~/utils';
@ -148,6 +149,7 @@ export default class NcUpgrader {
{ name: '0108002', handler: ncXcdbLTARUpgrader }, { name: '0108002', handler: ncXcdbLTARUpgrader },
{ name: '0111002', handler: ncXcdbLTARIndexUpgrader }, { name: '0111002', handler: ncXcdbLTARIndexUpgrader },
{ name: '0111005', handler: ncXcdbCreatedAndUpdatedSystemFieldsUpgrader }, { name: '0111005', handler: ncXcdbCreatedAndUpdatedSystemFieldsUpgrader },
{ name: '0225002', handler: ncDatasourceDecrypt },
]; ];
} }
} }

2
packages/nocodb/src/version-upgrader/ncFilterUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0100002_ncFilterUpgrader.ts

@ -1,4 +1,4 @@
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
import View from '~/models/View'; import View from '~/models/View';
import Hook from '~/models/Hook'; import Hook from '~/models/Hook';

6
packages/nocodb/src/version-upgrader/ncAttachmentUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0101002_ncAttachmentUpgrader.ts

@ -1,10 +1,10 @@
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import { throwTimeoutError } from './ncUpgradeErrors';
import type { XKnex } from '~/db/CustomKnex'; import type { XKnex } from '~/db/CustomKnex';
import type { Knex } from 'knex'; import type { Knex } from 'knex';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
// import type { XKnex } from '~/db/sql-data-mapper'; // import type { XKnex } from '~/db/sql-data-mapper';
import type { SourceType } from 'nocodb-sdk'; import type { SourceType } from 'nocodb-sdk';
import { throwTimeoutError } from '~/version-upgrader/ncUpgradeErrors';
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import Model from '~/models/Model'; import Model from '~/models/Model';
import Source from '~/models/Source'; import Source from '~/models/Source';
@ -46,7 +46,7 @@ function getTnPath(knex: XKnex, tb: Model) {
} }
export default async function ({ ncMeta }: NcUpgraderCtx) { export default async function ({ ncMeta }: NcUpgraderCtx) {
const sources: SourceType[] = await ncMeta.knexConnection(MetaTable.BASES); const sources: SourceType[] = await ncMeta.knexConnection(MetaTable.SOURCES);
for (const _base of sources) { for (const _base of sources) {
const source = new Source(_base); const source = new Source(_base);

6
packages/nocodb/src/version-upgrader/ncAttachmentUpgrader_0104002.ts → packages/nocodb/src/version-upgrader/upgraders/0104002_ncAttachmentUpgrader.ts

@ -1,9 +1,9 @@
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import { throwTimeoutError } from './ncUpgradeErrors';
import type { XKnex } from '~/db/CustomKnex'; import type { XKnex } from '~/db/CustomKnex';
import type { Knex } from 'knex'; import type { Knex } from 'knex';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import type { SourceType } from 'nocodb-sdk'; import type { SourceType } from 'nocodb-sdk';
import { throwTimeoutError } from '~/version-upgrader/ncUpgradeErrors';
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import Model from '~/models/Model'; import Model from '~/models/Model';
import Source from '~/models/Source'; import Source from '~/models/Source';
@ -37,7 +37,7 @@ function getTnPath(knex: XKnex, tb: Model) {
} }
export default async function ({ ncMeta }: NcUpgraderCtx) { export default async function ({ ncMeta }: NcUpgraderCtx) {
const sources: SourceType[] = await ncMeta.knexConnection(MetaTable.BASES); const sources: SourceType[] = await ncMeta.knexConnection(MetaTable.SOURCES);
for (const _base of sources) { for (const _base of sources) {
const source = new Source(_base); const source = new Source(_base);

2
packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts → packages/nocodb/src/version-upgrader/upgraders/0104004_ncFilterUpgrader.ts

@ -1,6 +1,6 @@
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import type { MetaService } from '~/meta/meta.service'; import type { MetaService } from '~/meta/meta.service';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import type { SelectOptionsType } from 'nocodb-sdk'; import type { SelectOptionsType } from 'nocodb-sdk';
import type { NcContext } from '~/interface/config'; import type { NcContext } from '~/interface/config';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';

2
packages/nocodb/src/version-upgrader/ncStickyColumnUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0105002_ncStickyColumnUpgrader.ts

@ -1,4 +1,4 @@
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
// before 0.104.3, display value column can be in any position in table // before 0.104.3, display value column can be in any position in table

2
packages/nocodb/src/version-upgrader/ncFilterUpgrader_0105003.ts → packages/nocodb/src/version-upgrader/upgraders/0105003_ncFilterUpgrader.ts

@ -1,6 +1,6 @@
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import type { MetaService } from '~/meta/meta.service'; import type { MetaService } from '~/meta/meta.service';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import type { NcContext } from '~/interface/config'; import type { NcContext } from '~/interface/config';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
import Column from '~/models/Column'; import Column from '~/models/Column';

2
packages/nocodb/src/version-upgrader/ncHookUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0105004_ncHookUpgrader.ts

@ -1,4 +1,4 @@
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
export default async function ({ ncMeta }: NcUpgraderCtx) { export default async function ({ ncMeta }: NcUpgraderCtx) {

4
packages/nocodb/src/version-upgrader/ncProjectConfigUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0107004_ncProjectConfigUpgrader.ts

@ -1,5 +1,5 @@
import CryptoJS from 'crypto-js'; import CryptoJS from 'crypto-js';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import { Source } from '~/models'; import { Source } from '~/models';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
@ -13,7 +13,7 @@ export default async function ({ ncMeta }: NcUpgraderCtx) {
const actions = []; const actions = [];
// Get all the base sources // Get all the base sources
const sources = await ncMeta.knexConnection(MetaTable.BASES); const sources = await ncMeta.knexConnection(MetaTable.SOURCES);
// Update the base config with the new secret key if we could decrypt the base config with the fallback secret key // Update the base config with the new secret key if we could decrypt the base config with the fallback secret key
for (const source of sources) { for (const source of sources) {

4
packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0108002_ncXcdbLTARUpgrader.ts

@ -1,7 +1,7 @@
import { RelationTypes, UITypes } from 'nocodb-sdk'; import { RelationTypes, UITypes } from 'nocodb-sdk';
import type { LinkToAnotherRecordColumn } from '~/models'; import type { LinkToAnotherRecordColumn } from '~/models';
import type { MetaService } from '~/meta/meta.service'; import type { MetaService } from '~/meta/meta.service';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import type { NcContext } from '~/interface/config'; import type { NcContext } from '~/interface/config';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
import NocoCache from '~/cache/NocoCache'; import NocoCache from '~/cache/NocoCache';
@ -163,7 +163,7 @@ async function upgradeBaseRelations(
// database to virtual relation and create an index for it // database to virtual relation and create an index for it
export default async function ({ ncMeta }: NcUpgraderCtx) { export default async function ({ ncMeta }: NcUpgraderCtx) {
// get all xcdb sources // get all xcdb sources
const sources = await ncMeta.knexConnection(MetaTable.BASES).where({ const sources = await ncMeta.knexConnection(MetaTable.SOURCES).where({
is_meta: 1, is_meta: 1,
}); });

4
packages/nocodb/src/version-upgrader/ncXcdbLTARIndexUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0111002_ncXcdbLTARIndexUpgrader.ts

@ -2,7 +2,7 @@ import { Logger } from '@nestjs/common';
import { RelationTypes, UITypes } from 'nocodb-sdk'; import { RelationTypes, UITypes } from 'nocodb-sdk';
import type { LinkToAnotherRecordColumn } from '~/models'; import type { LinkToAnotherRecordColumn } from '~/models';
import type { MetaService } from '~/meta/meta.service'; import type { MetaService } from '~/meta/meta.service';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import type { NcContext } from '~/interface/config'; import type { NcContext } from '~/interface/config';
import { MetaTable } from '~/utils/globals'; import { MetaTable } from '~/utils/globals';
import { Source } from '~/models'; import { Source } from '~/models';
@ -151,7 +151,7 @@ export default async function ({ ncMeta }: NcUpgraderCtx) {
); );
// get all xcdb sources // get all xcdb sources
const sources = await ncMeta.knexConnection(MetaTable.BASES).where({ const sources = await ncMeta.knexConnection(MetaTable.SOURCES).where({
is_meta: 1, is_meta: 1,
}); });

4
packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts → packages/nocodb/src/version-upgrader/upgraders/0111005_ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts

@ -1,5 +1,5 @@
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import type { NcUpgraderCtx } from './NcUpgrader'; import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import type { MetaService } from '~/meta/meta.service'; import type { MetaService } from '~/meta/meta.service';
import type { Base } from '~/models'; import type { Base } from '~/models';
import Noco from '~/Noco'; import Noco from '~/Noco';
@ -370,7 +370,7 @@ async function upgradeModels({
// database to virtual relation and create an index for it // database to virtual relation and create an index for it
export default async function ({ ncMeta }: NcUpgraderCtx) { export default async function ({ ncMeta }: NcUpgraderCtx) {
// get all xcdb sources // get all xcdb sources
const sources = await ncMeta.knexConnection(MetaTable.BASES).condition({ const sources = await ncMeta.knexConnection(MetaTable.SOURCES).condition({
_or: [ _or: [
{ {
is_meta: { is_meta: {

119
packages/nocodb/src/version-upgrader/upgraders/0225002_ncDatasourceDecrypt.ts

@ -0,0 +1,119 @@
import CryptoJS from 'crypto-js';
import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader';
import { MetaTable, RootScopes } from '~/utils/globals';
const logger = {
log: (message: string) => {
console.log(`[0225002_ncDatasourceDecrypt ${Date.now()}] ` + message);
},
error: (message: string) => {
console.error(`[0225002_ncDatasourceDecrypt ${Date.now()}] ` + message);
},
};
const decryptConfig = async (encryptedConfig: string, secret: string) => {
if (!encryptedConfig) return encryptedConfig;
const decryptedVal = CryptoJS.AES.decrypt(encryptedConfig, secret).toString(
CryptoJS.enc.Utf8,
);
// validate by parsing JSON
try {
JSON.parse(decryptedVal);
} catch {
throw new Error('Config decryption failed');
}
return decryptedVal;
};
// decrypt datasource details in source table and integration table
export default async function ({ ncMeta }: NcUpgraderCtx) {
let encryptionKey = process.env.NC_AUTH_JWT_SECRET;
if (!encryptionKey) {
encryptionKey = (
await ncMeta.metaGet(RootScopes.ROOT, RootScopes.ROOT, MetaTable.STORE, {
key: 'nc_auth_jwt_secret',
})
)?.value;
}
// if encryption key is same as previous, just update is_encrypted flag and return
if (
process.env.NC_CONNECTION_ENCRYPT_KEY &&
process.env.NC_CONNECTION_ENCRYPT_KEY === encryptionKey
) {
logger.log('Encryption key is same as previous. Skipping decryption');
await ncMeta.knexConnection(MetaTable.SOURCES).update({
is_encrypted: true,
});
await ncMeta.knexConnection(MetaTable.INTEGRATIONS).update({
is_encrypted: true,
});
return;
}
// if encryption key is not present, return
if (!encryptionKey) {
throw Error('Encryption key not found');
}
// get all external sources
const sources = await ncMeta.knexConnection(MetaTable.SOURCES);
const passed = [];
// iterate, decrypt and update
for (const source of sources) {
if (source?.config) {
try {
const decrypted = await decryptConfig(source.config, encryptionKey);
await ncMeta
.knexConnection(MetaTable.SOURCES)
.update({
config: decrypted,
})
.where('id', source.id);
passed.push(true);
} catch (e) {
logger.error(`Failed to decrypt source ${source.id}`);
passed.push(false);
}
}
}
// get all integrations
const integrations = await ncMeta.knexConnection(MetaTable.INTEGRATIONS);
// iterate, decrypt and update
for (const integration of integrations) {
if (integration?.config) {
try {
const decrypted = await decryptConfig(
integration.config,
encryptionKey,
);
await ncMeta
.knexConnection(MetaTable.INTEGRATIONS)
.update({
config: decrypted,
})
.where('id', integration.id);
passed.push(true);
} catch (e) {
logger.error(`Failed to decrypt integration ${integration.id}`);
passed.push(false);
}
}
}
// if all failed, log and exit
if (passed.length > 0 && passed.every((v) => !v)) {
throw new Error(
`Failed to decrypt any source or integration. Please configure correct encryption key.`,
);
}
logger.log(`Decrypted ${passed.length} sources and integrations`);
}

61
packages/nocodb/webpack.cli.config.js

@ -0,0 +1,61 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('./build-utils/resolveTsAliases');
module.exports = {
entry: './src/cli.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
optimization: {
minimize: true,
minimizer: [
new TerserPlugin({
extractComments: false,
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('tsconfig.json')),
},
mode: 'production',
output: {
filename: 'cli.js',
path: path.resolve(__dirname, '..', 'nc-secret-mgr', 'src/nocodb'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new webpack.EnvironmentPlugin(['EE']),
new webpack.BannerPlugin({
banner: 'This is a generated file. Do not edit',
entryOnly:true
}),
],
target: 'node',
};

1070
pnpm-lock.yaml

File diff suppressed because it is too large Load Diff

3
pnpm-workspace.yaml

@ -3,4 +3,5 @@ packages:
- 'packages/nc-gui' - 'packages/nc-gui'
- 'packages/nc-mail-templates' - 'packages/nc-mail-templates'
- 'packages/nocodb' - 'packages/nocodb'
- 'tests/playwright' - 'tests/playwright'
- 'packages/nc-secret-mgr'

14
scripts/updateCliVersion.js

@ -0,0 +1,14 @@
const fs = require('fs')
const path = require('path')
const packageJsonPath = path.join(__dirname, '..', 'packages', 'nc-secret-mgr', 'package.json')
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
if (!process.env.targetVersion) {
console.error('Error: targetVersion environment variable is not defined.');
process.exit(1);
}
packageJson.version = process.env.targetVersion
fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2))

2
tests/playwright/pages/Dashboard/Details/ErdPage.ts

@ -39,6 +39,8 @@ export class ErdPage extends BasePage {
columnName?: string; columnName?: string;
columnNameShouldNotExist?: string; columnNameShouldNotExist?: string;
}) { }) {
await this.get().locator(`.nc-erd-table-node-${tableName}`).scrollIntoViewIfNeeded();
await this.get().locator(`.nc-erd-table-node-${tableName}`).waitFor({ state: 'visible' }); await this.get().locator(`.nc-erd-table-node-${tableName}`).waitFor({ state: 'visible' });
if (columnName) { if (columnName) {
await this.get().locator(`.nc-erd-table-node-${tableName}-column-${columnName}`).waitFor({ state: 'visible' }); await this.get().locator(`.nc-erd-table-node-${tableName}-column-${columnName}`).waitFor({ state: 'visible' });

Loading…
Cancel
Save