feat: docs, action for deployment and other corrections

packages/nc-secret-cli/.npmignore

@@ -0,0 +1,5 @@
+tsconfig.json
+webpack.config.js
+src
+node_modules
+scripts

packages/nc-secret-cli/dist/cli.js.LICENSE.txt

@@ -0,0 +1,68 @@
+/*!
+ * mime-db
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * mime-types
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*! *****************************************************************************
+Copyright (C) Microsoft. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */
+
+/**
+ * @copyright Copyright (c) 2021, Xgene Cloud Ltd
+ *
+ * @author Naveen MR <oof1lab@gmail.com>
+ * @author Pranav C Balan <pranavxc@gmail.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/** @preserve
+	 * Counter block mode compatible with  Dr Brian Gladman fileenc.c
+	 * derived from CryptoJS.mode.CTR
+	 * Jan Hruby jhruby.web@gmail.com
+	 */
+
+/** @preserve
+	(c) 2012 by Cédric Mesnil. All rights reserved.
+
+	Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+	    - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+	    - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+	*/

packages/nc-secret-cli/src/core/NcConfig.ts

@@ -1,7 +1,7 @@
 import * as path from 'path';
 import fs from 'fs';
 import { promisify } from 'util';
-const { DriverClient, getToolDir, metaUrlToDbConfig } = require( '../nocodb/cli');
+const { DriverClient, getToolDir, metaUrlToDbConfig, prepareEnv } = require( '../nocodb/cli');
 
 export class NcConfig {
   meta: {
@@ -28,6 +28,8 @@ export class NcConfig {
       metaUrl?: string;
       metaJson?: string;
       metaJsonFile?: string;
+      databaseUrlFile?: string;
+      databaseUrl?: string;
     };
     secret?: string;
     credentialSecret?: string;
@@ -86,6 +88,13 @@ export const getNocoConfig = (options: {
   databaseUrl?: string;
   databaseUrlFile?: string;
 } ={}) =>{
+  // check for JDBC url specified in env or options
+  await prepareEnv({
+    databaseUrl: options.ncDatabaseUrl || process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
+    databaseUrlFile:  options.ncDatabaseUrlFile || process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
+  })
+
+  // create NocoConfig using utility method which works similar to Nocodb NcConfig with only meta db config
   return NcConfig.create({
     meta: {
       metaUrl: process.env.NC_DB || options.ncDb,

packages/nc-secret-cli/src/core/SecretManager.ts

@@ -6,7 +6,7 @@ export class SecretManager {
 
   private sqlClient;
 
-  constructor(private oldSecret: string, private newSecret: string, private config: any) {
+  constructor(private prevSecret: string, private newSecret: string, private config: any) {
     this.sqlClient = SqlClientFactory.create(this.config.meta.db);
   }
 
@@ -54,7 +54,7 @@ export class SecretManager {
       try {
         const decrypted = decryptPropIfRequired({
           data: source,
-          secret: this.oldSecret,
+          secret: this.prevSecret,
           prop: 'config'
         });
         isValid = true;
@@ -68,7 +68,7 @@ export class SecretManager {
       try {
         const decrypted = decryptPropIfRequired({
           data: integration,
-          secret: this.oldSecret,
+          secret: this.prevSecret,
           prop: 'config'
         });
         isValid = true;

packages/nc-secret-cli/src/index.spec.ts

@@ -6,10 +6,10 @@ describe('Index', () => {
 
 describe('index.ts', () => {
   it('should parse the arguments and options correctly', () => {
-    const argv = ['node', 'index.ts', 'oldSecret', 'newSecret', '--nc-db','test_db_url', '--database-url', 'test_db_url', '-o', 'oldSecret', '-n', 'newSecret'];
+    const argv = ['node', 'index.ts', 'prevSecret', 'newSecret', '--nc-db','test_db_url', '--database-url', 'test_db_url', '-o', 'prevSecret', '-n', 'newSecret'];
     program.parse(argv);
-    expect(program.opts().oldSecret).to.equal('oldSecret');
+    expect(program.opts().prev).to.equal('prevSecret');
-    expect(program.opts().newSecret).to.equal('newSecret');
+    expect(program.opts().new).to.equal('newSecret');
     expect(program.opts().ncDb).to.equal('test_db_url');
     expect(program.opts().databaseUrl).to.equal('test_db_url');
   });

packages/nc-secret-cli/src/index.ts

@@ -1,9 +1,9 @@
 import figlet from "figlet";
 import { Command } from 'commander';
-import { getNocoConfig } from "./core/NcConfig";
+import { getNocoConfig } from "./core";
-import { SecretManager } from "./core/SecretManager";
+import { SecretManager } from "./core";
-import { NcError } from "./core/NcError";
+import { NcError } from "./core";
-import { NcLogger } from "./core/NcLogger";
+import { NcLogger } from "./core";
 
 console.log(figlet.textSync("NocoDB Secret CLI"));
 
@@ -12,7 +12,7 @@ const program = new Command();
 program
   .version('1.0.0')
   .description('NocoDB Secret CLI')
-  .arguments('<oldSecret> <newSecret>')
+  .arguments('<prevSecret> <newSecret>')
   .option('--nc-db <char>', 'NocoDB  connection database url, equivalent to NC_DB env variable')
   .option('--nc-db-json <char>', 'NocoDB connection database json, equivalent to NC_DB_JSON env variable')
   .option('--nc-db-json-file <char>', 'NocoDB connection database json file path, equivalent to NC_DB_JSON_FILE env variable')

packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md

@@ -15,11 +15,12 @@ For production use cases, it is crucial to set all environment variables marked
 | `NC_DB_JSON_FILE` | No        | A path to a knex connection JSON file can be used to specify the database connection, as an alternative to `NC_DB`. | |
 | `DATABASE_URL` | No        | A [JDBC URL string](https://jdbc.postgresql.org/documentation/use/#connecting-to-the-database) can be used for the database connection instead of `NC_DB`. | |
 | `DATABASE_URL_FILE` | No        | A path to a file containing a JDBC URL can be specified for the database connection as an alternative to `NC_DB`. | |
+| `NC_KEY_CREDENTIAL_ENCRYPT` | No        | The key used to encrypt the credentials of external databases. <br><br/> **Warning:** Changing this variable may break the application. If you must change it, use the CLI as described in the [NocoDB Secret CLI documentation](/data-sources/updating-secret). | Keep connection credentials as plain text in the database if not set. |
 
 ## Authentication
 | Variable | Mandatory | Description | If Not Set |
 | -------- |-----------| ----------- | ---------- |
-| `NC_AUTH_JWT_SECRET` | Yes       | This JWT secret is utilized for generating authentication tokens and encrypting credentials for external databases. | A random secret will be generated automatically. |
+| `NC_AUTH_JWT_SECRET` | Yes       | This JWT secret is utilized for generating authentication tokens. | A random secret will be generated automatically. |
 | `NC_JWT_EXPIRES_IN` | No        | Specifies the expiration time for JWT tokens. | Defaults to `10h`. |
 | `NC_GOOGLE_CLIENT_ID` | No        | Google client ID required to activate Google authentication. | |
 | `NC_GOOGLE_CLIENT_SECRET` | No        | Google client secret required to activate Google authentication. | |

packages/noco-docs/docs/100.data-sources/050.updating-secret.md

@@ -0,0 +1,60 @@
+---
+title: 'Updating Secrets'
+description: 'Learn how to update secrets in NocoDB using the nc-secret-cli package.'
+tags: ['Secrets', 'nc-secret-cli', 'Update', 'Security']
+keywords: ['NocoDB secrets', 'nc-secret-cli', 'Update', 'Security']
+---
+
+## Updating Secrets
+
+To update a secret in NocoDB, you can use the `nc-secret-cli` package. Follow the steps below to update a secret:
+
+### Using the Command Line Interface (CLI)
+
+1. Install the `nc-secret-cli` package if you haven't already. You can do this by running the following command in your terminal:
+
+   ```bash
+   npm install -g nc-secret-cli
+   ```
+
+2. Once the package is installed, you can update a secret by running the following command:
+
+   ```bash
+   NC_DB="pg://host:port?u=user&p=password&d=database" nc-secret-cli update --prev <previous-secret> --new <new-secret>
+   ```
+   
+   OR
+
+   ```bash
+   NC_DB="pg://host:port?u=user&p=password&d=database" nc-secret-cli <previous-secret> --new <new-secret
+   ```
+
+   Replace `<prev-secret>` with the name of the secret you used previously, and `<new-secret>` with the new value of the secret.
+
+3. After running the command, the secret will be updated in NocoDB.
+
+### Using Executables
+
+Alternatively, you can use the `nc-secret-cli` executable to update secrets.
+
+1. Download the `nc-secret-cli` executable from the [NocoDB website](https://github.com/nocodb/nc-secret-cli/releases/latest).
+4. Run the executable using the following command:
+
+   ```bash
+   NC_DB="pg://host:port?u=user&p=password&d=database" ./nc-secret-macos-arm64 update --prev <previous-secret> --new <new-secret>
+   ```
+
+   Replace `<prev-secret>` with the name of the secret you used previously, and `<new-secret>` with the new value of the secret.
+
+5. After running the command, the secret will be updated in NocoDB.
+
+
+Note: All environment variables are supported, including `NC_DB`, `NC_DB_JSON`, `NC_DB_JSON_FILE`, `DATABASE_URL`, and `DATABASE_URL_FILE`. You can use any of these variables to specify your database connection. Alternately you can use following equivalent parameters.
+
+| Environment Variable | CLI Parameter |
+| --------------------- | -------------- |
+| `NC_DB`               | `--nc-db`      |
+| `NC_DB_JSON`          | `--nc-db-json` |
+| `NC_DB_JSON_FILE`     | `--nc-db-json-file` |
+| `DATABASE_URL`        | `--database-url` |
+| `DATABASE_URL_FILE`   | `--database-url-file` |

packages/nocodb/src/helpers/initDataSourceEncryption.ts

@@ -28,6 +28,8 @@ export default async function initDataSourceEncryption(_ncMeta = Noco.ncMeta) {
         qb.where('is_local', false).orWhereNull('is_local');
       });
 
+    const isAtleastOneSourceEncrypted = false;
+
     for (const source of sources) {
       // encrypt the data source
       await ncMeta.metaUpdate(

packages/nocodb/src/models/Integration.ts

@@ -12,7 +12,7 @@ import {
 } from '~/utils/modelUtils';
 import {
   decryptPropIfRequired,
-  encryptPropIfRequired,
+  encryptPropIfRequired, isEncryptionRequired,
   partialExtract,
 } from '~/utils';
 import { PagedResponseImpl } from '~/helpers/PagedResponse';
@@ -30,6 +30,7 @@ export default class Integration implements IntegrationType {
   meta?: any;
   created_by?: string;
   sources?: Partial<SourceType>[];
+  is_encrypted?: BoolType;
 
   constructor(integration: Partial<IntegrationType>) {
     Object.assign(this, integration);
@@ -45,6 +46,7 @@ export default class Integration implements IntegrationType {
       created_at?;
       updated_at?;
       meta?: any;
+      is_encrypted?: BoolType;
     },
     ncMeta = Noco.ncMeta,
   ) {
@@ -57,12 +59,15 @@ export default class Integration implements IntegrationType {
       'meta',
       'created_by',
       'is_private',
+      'is_encrypted',
     ]);
 
     insertObj.config = encryptPropIfRequired({
       data: insertObj,
     });
 
+    insertObj.is_encrypted = isEncryptionRequired();
+
     if ('meta' in insertObj) {
       insertObj.meta = stringifyMetaProp(insertObj);
     }
@@ -101,6 +106,7 @@ export default class Integration implements IntegrationType {
     integration: IntegrationType & {
       meta?: any;
       deleted?: boolean;
+      is_encrypted?: boolean;
     },
     ncMeta = Noco.ncMeta,
   ) {
@@ -123,12 +129,14 @@ export default class Integration implements IntegrationType {
       'deleted',
       'config',
       'is_private',
+      'is_encrypted'
     ]);
 
     if (updateObj.config) {
       updateObj.config = encryptPropIfRequired({
         data: updateObj,
       });
+      updateObj.is_encrypted = isEncryptionRequired();
     }
 
     // type property is undefined even if not provided

packages/nocodb/src/models/Source.ts

@@ -29,6 +29,7 @@ import {
   decryptPropIfRequired,
   deepMerge,
   encryptPropIfRequired,
+  isEncryptionRequired,
   partialExtract,
 } from '~/utils';
 
@@ -52,6 +53,7 @@ export default class Source implements SourceType {
   fk_integration_id?: string;
   integration_config?: string;
   integration_title?: string;
+  is_encrypted?: boolean;
 
   constructor(source: Partial<SourceType>) {
     Object.assign(this, source);
@@ -68,6 +70,7 @@ export default class Source implements SourceType {
       created_at?;
       updated_at?;
       meta?: any;
+      is_encrypted?: any;
     },
     ncMeta = Noco.ncMeta,
   ) {
@@ -86,12 +89,15 @@ export default class Source implements SourceType {
       'is_schema_readonly',
       'is_data_readonly',
       'fk_integration_id',
+      'is_encrypted',
     ]);
 
     insertObj.config = encryptPropIfRequired({
       data: insertObj,
     });
 
+    insertObj.is_encrypted = isEncryptionRequired();
+
     if ('meta' in insertObj) {
       insertObj.meta = stringifyMetaProp(insertObj);
     }
@@ -125,6 +131,7 @@ export default class Source implements SourceType {
       meta?: any;
       deleted?: boolean;
       fk_sql_executor_id?: string;
+      is_encrypted?: boolean;
     },
     ncMeta = Noco.ncMeta,
   ) {
@@ -148,12 +155,14 @@ export default class Source implements SourceType {
       'is_schema_readonly',
       'is_data_readonly',
       'fk_integration_id',
+      'is_encrypted',
     ]);
 
     if (updateObj.config) {
       updateObj.config = encryptPropIfRequired({
         data: updateObj,
       });
+      updateObj.is_encrypted = isEncryptionRequired();
     }
 
     // type property is undefined even if not provided
@@ -226,7 +235,9 @@ export default class Source implements SourceType {
     args: { baseId: string },
     ncMeta = Noco.ncMeta,
   ): Promise<Source[]> {
-    const cachedList = await NocoCache.getList(CacheScope.SOURCE, [args.baseId]);
+    const cachedList = await NocoCache.getList(CacheScope.SOURCE, [
+      args.baseId,
+    ]);
     let { list: sourceDataList } = cachedList;
     const { isNoneList } = cachedList;
     if (!isNoneList && !sourceDataList.length) {

packages/nocodb/src/utils/encryptDecrypt.ts

@@ -1,8 +1,10 @@
 import CryptoJS from 'crypto-js';
 
-
 export const credentialEncyptSecret = process.env.NC_KEY_CREDENTIAL_ENCRYPT;
 
+export const isEncryptionRequired = (secret = credentialEncyptSecret) => {
+  return !!secret;
+};
 export const encryptPropIfRequired = ({
   data,
   prop = 'config',
@@ -20,10 +22,7 @@ export const encryptPropIfRequired = ({
     return data[prop];
   }
 
-  return CryptoJS.AES.encrypt(
+  return CryptoJS.AES.encrypt(JSON.stringify(data[prop]), secret).toString();
-    JSON.stringify(data[prop]),
-    secret,
-  ).toString();
 };
 
 export const decryptPropIfRequired = ({

packages/nocodb/src/utils/nc-config/helpers.ts

@@ -13,7 +13,10 @@ import {
 import { DriverClient } from './interfaces';
 import type { Connection, DbConfig } from './interfaces';
 
-export async function prepareEnv() {
+export async function prepareEnv({
+databaseUrlFile = process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
+databaseUrl = process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
+                                 } = {}) {
   if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
     const database_url = await promisify(fs.readFile)(
       process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,

scripts/pkg-secret-cli-executable/index.js

@@ -1,13 +0,0 @@
-process.env.NC_BINARY_BUILD = 'true';
-(async () => {
-  try {
-    const app = require('express')();
-    const {Noco} = require("nocodb");
-    const port = process.env.PORT || 8080;
-    const httpServer = app.listen(port);
-    app.use(await Noco.init({}, httpServer, app));
-    console.log(`Visit : localhost:${port}/dashboard`)
-  } catch(e) {
-    console.log(e)
-  }
-})()

scripts/updateCliVersion.js

@@ -0,0 +1,8 @@
+const fs = require('fs')
+const path = require('path')
+
+const packageJsonPath =path.join(__dirname, '..', 'packages', 'nc-secret-cli', 'package.json')
+
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
+packageJson.version = process.env.targetVersion
+fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, 0, 2))