feat: cli for updating secret

.gitignore

@@ -96,3 +96,5 @@ test_noco.db
 httpbin
 .run/test-debug.run.xml
 
+/packages/nc-secret-cli/dist/index.js
+/packages/nc-secret-cli/dist/index.js.map

packages/nc-secret-cli/src/core/SecretManager.ts

@@ -0,0 +1,112 @@
+import { SqlClientFactory, MetaTable, decryptPropIfRequired, encryptPropIfRequired } from 'nocodb';
+
+export class SecretManager {
+
+  private sqlClient;
+
+  constructor(private oldSecret: string, private newSecret: string, private config: any) {
+    this.sqlClient = SqlClientFactory.create(this.config.meta.db);
+  }
+
+
+  // validate config by checking if database config is valid
+  async validateConfig() {
+    // use the sqlClientFactory to create a new sql client and then use testConnection to test the connection
+    const isValid = await this.sqlClient.testConnection();
+    if (!isValid) {
+      throw new Error('Invalid database configuration');
+    }
+  }
+
+
+  async validateAndExtract() {
+    const sources = await this.sqlClient.knex(MetaTable.SOURCES).where(qb => {
+      qb.where('is_meta', false).orWhere('is_meta', null)
+    });
+
+    const integrations = await this.sqlClient.knex(MetaTable.INTEGRATIONS).where(qb => {
+      qb.where('is_meta', false).orWhere('is_meta', null)
+    });
+
+    const sourcesToUpdate: Record<string, any>[] = [];
+    const integrationsToUpdate: Record<string, any>[] = [];
+
+
+    let isValid = false;
+    for (const source of sources) {
+      try {
+        const decrypted = decryptPropIfRequired({
+          data: source,
+          secret: this.oldSecret,
+          prop: 'config'
+        });
+        isValid = true;
+        sourcesToUpdate.push({ ...source, config: decrypted });
+      } catch (e) {
+        console.log(e);
+      }
+    }
+
+    for (const integration of integrations) {
+      try {
+        const decrypted = decryptPropIfRequired({
+          data: integration,
+          secret: this.oldSecret,
+          prop: 'config'
+        });
+        isValid = true;
+        integrationsToUpdate.push({ ...integration, config: decrypted });
+      } catch (e) {
+        console.log(e);
+      }
+    }
+
+    // if all of the decyptions are failed then throw an error
+    if (!isValid) {
+      throw new Error('Invalid old secret or no sources/integrations found');
+    }
+
+
+    return { sourcesToUpdate, integrationsToUpdate };
+  }
+
+
+  async updateSecret(
+    sourcesToUpdate: Record<string, any>[],
+    integrationsToUpdate: Record<string, any>[]
+  ) {
+    // start transaction
+    const transaction = await this.sqlClient.transaction();
+
+    try {
+      //  update sources
+      for (const source of sourcesToUpdate) {
+        await transaction(MetaTable.SOURCES).update({
+          config: encryptPropIfRequired({
+            data: source,
+            secret: this.newSecret,
+            prop: 'config'
+          })
+        }).where('id', source.id);
+      }
+
+      // update integrations
+      for (const integration of integrationsToUpdate) {
+        await transaction(MetaTable.INTEGRATIONS).update({
+          config: encryptPropIfRequired({
+            data: integration,
+            secret: this.newSecret,
+            prop: 'config'
+          })
+        }).where('id', integration.id);
+      }
+
+      await transaction.commit();
+
+    } catch (e) {
+      console.log(e);
+      await transaction.rollback();
+      throw e;
+    }
+  }
+}

packages/nc-secret-cli/src/index.ts

@@ -0,0 +1,43 @@
+import figlet from "figlet";
+
+console.log(figlet.textSync("Nocodb Secret CLI"));
+
+import { Command } from 'commander';
+import { getNocoConfig } from "./nc-config";
+import { SecretManager } from "./core/SecretManager";
+
+const program = new Command();
+
+program
+    .version('1.0.0')
+    .description('NocoDB Secret CLI')
+    .arguments('<oldSecret> <newSecret>')
+    .action(async (key, value) => {
+
+        const config = await getNocoConfig();
+
+        if (!key || !value) {
+            console.error('Error: Both key and value are required.');
+            program.help();
+        } else {
+            const secretManager = new SecretManager(key, value, config);
+
+            // validate meta db config which is resolved from env variables
+            await secretManager.validateConfig();
+
+            // validate old secret
+            const { sourcesToUpdate, integrationsToUpdate } = await secretManager.validateAndExtract();
+
+
+            // update sources and integrations
+            await secretManager.updateSecret(sourcesToUpdate, integrationsToUpdate);
+
+        }
+    });
+
+
+
+// Add error handling
+program.exitOverride();
+
+program.parse(process.argv);

packages/nc-secret-cli/src/nc-config/NcConfig.ts

@@ -0,0 +1,95 @@
+import * as path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { getToolDir, metaUrlToDbConfig } from './helpers';
+import { DriverClient } from './interfaces';
+import type { DbConfig } from './interfaces';
+import { SqlClientFactory } from 'nocodb';
+
+export class NcConfig {
+  meta: {
+    db: DbConfig;
+  } = {
+    db: {
+      client: DriverClient.SQLITE,
+      connection: {
+        filename: 'noco.db',
+      },
+    },
+  };
+
+  toolDir: string;
+
+  credentialSecret?: string;
+
+  private constructor() {
+    this.toolDir = getToolDir();
+  }
+
+  public static async create(param: {
+    meta: {
+      metaUrl?: string;
+      metaJson?: string;
+      metaJsonFile?: string;
+    };
+    secret?: string;
+    credentialSecret?: string;
+  }): Promise<NcConfig> {
+    const { meta, secret } =
+      param;
+
+    const ncConfig = new NcConfig();
+
+
+    ncConfig.credentialSecret = param.credentialSecret;
+
+
+    if (ncConfig.meta?.db?.connection?.filename) {
+      ncConfig.meta.db.connection.filename = path.join(
+        ncConfig.toolDir,
+        ncConfig.meta.db.connection.filename,
+      );
+    }
+
+      if (meta?.metaUrl) {
+        ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl);
+      } else if (meta?.metaJson) {
+        ncConfig.meta.db = JSON.parse(meta.metaJson);
+      } else if (meta?.metaJsonFile) {
+        if (!(await promisify(fs.exists)(meta.metaJsonFile))) {
+          throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`);
+        }
+        const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, {
+          encoding: 'utf8',
+        });
+        ncConfig.meta.db = JSON.parse(fileContent);
+      }
+
+
+    return ncConfig;
+  }
+
+  public static async createByEnv(): Promise<NcConfig> {
+    return NcConfig.create({
+      meta: {
+        metaUrl: process.env.NC_DB,
+        metaJson: process.env.NC_DB_JSON,
+        metaJsonFile: process.env.NC_DB_JSON_FILE,
+      },
+      secret: process.env.NC_AUTH_JWT_SECRET,
+      credentialSecret: process.env.NC_KEY_CREDENTIAL_ENCRYPT,
+    });
+  }
+}
+
+export const getNocoConfig = () =>{
+  return NcConfig.create({
+    meta: {
+      metaUrl: process.env.NC_DB,
+      metaJson: process.env.NC_DB_JSON,
+      metaJsonFile: process.env.NC_DB_JSON_FILE,
+    },
+    secret: process.env.NC_AUTH_JWT_SECRET,
+    credentialSecret: process.env.NC_KEY_CREDENTIAL_ENCRYPT,
+  });
+}

packages/nc-secret-cli/src/nc-config/constants.ts

@@ -0,0 +1,86 @@
+export const driverClientMapping = {
+  mysql: 'mysql2',
+  mariadb: 'mysql2',
+  postgres: 'pg',
+  postgresql: 'pg',
+  sqlite: 'sqlite3',
+  mssql: 'mssql',
+};
+
+export const defaultClientPortMapping = {
+  mysql: 3306,
+  mysql2: 3306,
+  postgres: 5432,
+  pg: 5432,
+  mssql: 1433,
+};
+
+export const defaultConnectionConfig: any = {
+  // https://github.com/knex/knex/issues/97
+  // timezone: process.env.NC_TIMEZONE || 'UTC',
+  dateStrings: true,
+};
+
+// default knex options
+export const defaultConnectionOptions = {
+  pool: {
+    min: 0,
+    max: 10,
+  },
+};
+
+export const avoidSSL = [
+  'localhost',
+  '127.0.0.1',
+  'host.docker.internal',
+  '172.17.0.1',
+];
+
+export const knownQueryParams = [
+  {
+    parameter: 'database',
+    aliases: ['d', 'db'],
+  },
+  {
+    parameter: 'password',
+    aliases: ['p'],
+  },
+  {
+    parameter: 'user',
+    aliases: ['u'],
+  },
+  {
+    parameter: 'title',
+    aliases: ['t'],
+  },
+  {
+    parameter: 'keyFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'certFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'caFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'ssl',
+    aliases: [],
+  },
+  {
+    parameter: 'options',
+    aliases: ['opt', 'opts'],
+  },
+];
+
+export enum DriverClient {
+  MYSQL = 'mysql2',
+  MYSQL_LEGACY = 'mysql',
+  MSSQL = 'mssql',
+  PG = 'pg',
+  SQLITE = 'sqlite3',
+  SNOWFLAKE = 'snowflake',
+  DATABRICKS = 'databricks',
+}

packages/nc-secret-cli/src/nc-config/helpers.ts

@@ -0,0 +1,326 @@
+import fs from 'fs';
+import { URL } from 'url';
+import { promisify } from 'util';
+import parseDbUrl from 'parse-database-url';
+import {
+  avoidSSL,
+  defaultClientPortMapping,
+  defaultConnectionConfig,
+  defaultConnectionOptions,
+  driverClientMapping,
+  knownQueryParams,
+} from './constants';
+import { DriverClient } from './interfaces';
+import type { Connection, DbConfig } from './interfaces';
+
+export async function prepareEnv() {
+  if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
+    const database_url = await promisify(fs.readFile)(
+      (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) as string,
+      'utf-8',
+    );
+    process.env.NC_DB = jdbcToXcUrl(database_url);
+  } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
+    process.env.NC_DB = jdbcToXcUrl(
+      (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) as string,
+    );
+  }
+}
+
+export function getToolDir() {
+  return process.env.NC_TOOL_DIR || process.cwd();
+}
+
+export function jdbcToXcConfig(url: string): DbConfig {
+  // drop the jdbc prefix
+  url.replace(/^jdbc:/, '');
+
+  const config = parseDbUrl(url);
+
+  const parsedConfig: Connection = {};
+
+  for (const [key, value] of Object.entries(config)) {
+    const fnd = knownQueryParams.find(
+      (param) => param.parameter === key || param.aliases.includes(key),
+    );
+    if (fnd) {
+      parsedConfig[fnd.parameter] = value;
+    } else {
+      parsedConfig[key] = value;
+    }
+  }
+
+  if (!parsedConfig?.port) {
+    parsedConfig.port =
+      defaultClientPortMapping[
+        driverClientMapping[parsedConfig.driver as DriverClient] || parsedConfig.driver
+      ];
+  }
+
+  const { driver, ...connectionConfig } = parsedConfig;
+
+  const client = driverClientMapping[driver as DriverClient] || driver;
+
+  if (
+    client === 'pg' &&
+    !connectionConfig?.ssl &&
+    !avoidSSL.includes(connectionConfig.host as string)
+  ) {
+    connectionConfig.ssl = true;
+  }
+
+  return {
+    client: client,
+    connection: {
+      ...connectionConfig,
+    },
+  } as DbConfig;
+}
+
+export function jdbcToXcUrl(url: string): string {
+  // drop the jdbc prefix
+  url.replace(/^jdbc:/, '');
+
+  const config = parseDbUrl(url);
+
+  const parsedConfig: Connection = {};
+
+  for (const [key, value] of Object.entries(config)) {
+    const fnd = knownQueryParams.find(
+      (param) => param.parameter === key || param.aliases.includes(key),
+    );
+    if (fnd) {
+      parsedConfig[fnd.parameter] = value;
+    } else {
+      parsedConfig[key] = value;
+    }
+  }
+
+  if (!parsedConfig?.port) {
+    parsedConfig.port =
+      defaultClientPortMapping[
+        driverClientMapping[parsedConfig.driver as DriverClient] || parsedConfig.driver
+      ];
+  }
+
+  const { driver, host, port, database, user, password, ...extra } =
+    parsedConfig;
+
+  const extraParams: string[] = [];
+
+  for (const [key, value] of Object.entries(extra)) {
+    extraParams.push(`${key}=${encodeURIComponent(String(value))}`);
+  }
+
+  const res = `${driverClientMapping[driver as DriverClient] || driver}://${host}${
+    port ? `:${port}` : ''
+  }?${user ? `u=${encodeURIComponent(user)}&` : ''}${
+    password ? `p=${encodeURIComponent(password)}&` : ''
+  }${database ? `d=${encodeURIComponent(database)}&` : ''}${extraParams.join(
+    '&',
+  )}`;
+
+  return res;
+}
+
+export function xcUrlToDbConfig(
+  urlString: string,
+  key = '',
+  type?: string,
+): DbConfig {
+  const url = new URL(urlString);
+
+  let dbConfig: DbConfig;
+
+  if (url.protocol.startsWith('sqlite3')) {
+    dbConfig = {
+      client: 'sqlite3',
+      connection: {
+        client: 'sqlite3',
+        connection: {
+          filename:
+            url.searchParams.get('d') || url.searchParams.get('database'),
+        },
+        database: url.searchParams.get('d') || url.searchParams.get('database'),
+      },
+    } as any;
+  } else {
+    const parsedQuery = {};
+    for (const [key, value] of url.searchParams.entries()) {
+      const fnd = knownQueryParams.find(
+        (param) => param.parameter === key || param.aliases.includes(key),
+      );
+      if (fnd) {
+        parsedQuery[fnd.parameter] = value;
+      } else {
+        parsedQuery[key] = value;
+      }
+    }
+
+    dbConfig = {
+      client: url.protocol.replace(':', '') as DriverClient,
+      connection: {
+        ...parsedQuery,
+        host: url.hostname,
+        port: +url.port,
+      },
+      acquireConnectionTimeout: 600000,
+    };
+
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
+      dbConfig.connection.ssl = true;
+    }
+
+    if (
+      url.searchParams.get('keyFilePath') &&
+      url.searchParams.get('certFilePath') &&
+      url.searchParams.get('caFilePath')
+    ) {
+      dbConfig.connection.ssl = {
+        keyFilePath: url.searchParams.get('keyFilePath') as string,
+        certFilePath: url.searchParams.get('certFilePath') as string,
+        caFilePath: url.searchParams.get('caFilePath') as string,
+      };
+    }
+  }
+
+  /* TODO check if this is needed
+    if (config && !config.title) {
+      config.title =
+        url.searchParams.get('t') ||
+        url.searchParams.get('title') ||
+        this.generateRandomTitle();
+    }
+  */
+
+  Object.assign(dbConfig, {
+    meta: {
+      tn: 'nc_evolutions',
+      allSchemas:
+        !!url.searchParams.get('allSchemas') ||
+        !(url.searchParams.get('d') || url.searchParams.get('database')),
+      api: {
+        prefix: url.searchParams.get('apiPrefix') || '',
+        swagger: true,
+        type:
+          type ||
+          ((url.searchParams.get('api') || url.searchParams.get('a')) as any) ||
+          'rest',
+      },
+      dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
+      metaTables: 'db',
+      migrations: {
+        disabled: false,
+        name: 'nc_evolutions',
+      },
+    },
+  });
+
+  return dbConfig;
+}
+
+export async function metaUrlToDbConfig(urlString): Promise<DbConfig> {
+  const url = new URL(urlString);
+
+  let dbConfig: DbConfig;
+
+  if (url.protocol.startsWith('sqlite3')) {
+    const db = url.searchParams.get('d') || url.searchParams.get('database');
+    dbConfig = {
+      client: DriverClient.SQLITE,
+      connection: {
+        filename: db as string,
+      },
+      ...(db === ':memory:'
+        ? {
+            pool: {
+              min: 1,
+              max: 1,
+              // disposeTimeout: 360000*1000,
+              idleTimeoutMillis: 360000 * 1000,
+            },
+          }
+        : {}),
+    };
+  } else {
+    const parsedQuery = {};
+    for (const [key, value] of url.searchParams.entries()) {
+      const fnd = knownQueryParams.find(
+        (param) => param.parameter === key || param.aliases.includes(key),
+      );
+      if (fnd) {
+        parsedQuery[fnd.parameter] = value;
+      } else {
+        parsedQuery[key] = value;
+      }
+    }
+
+    dbConfig = {
+      client: url.protocol.replace(':', '') as DriverClient,
+      connection: {
+        ...defaultConnectionConfig,
+        ...parsedQuery,
+        host: url.hostname,
+        port: +url.port,
+      },
+      acquireConnectionTimeout: 600000,
+      ...defaultConnectionOptions,
+      ...(url.searchParams.has('search_path')
+        ? {
+            searchPath: url.searchParams.get('search_path')?.split(','),
+          }
+        : {}),
+    };
+
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
+      dbConfig.connection.ssl = true;
+    }
+  }
+
+  url.searchParams.forEach((_value, key) => {
+    let value: any = _value;
+    if (value === 'true') {
+      value = true;
+    } else if (value === 'false') {
+      value = false;
+    } else if (/^\d+$/.test(value)) {
+      value = +value;
+    }
+    // todo: implement config read from JSON file or JSON env val read
+    if (
+      !['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes(
+        key,
+      )
+    ) {
+      key.split('.').reduce((obj, k, i, arr) => {
+        return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
+      }, dbConfig);
+    }
+  });
+
+  if (
+    dbConfig?.connection?.ssl &&
+    typeof dbConfig?.connection?.ssl === 'object'
+  ) {
+    if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
+      dbConfig.connection.ssl.ca = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.caFilePath;
+    }
+    if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
+      dbConfig.connection.ssl.key = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.keyFilePath;
+    }
+    if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) {
+      dbConfig.connection.ssl.cert = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.certFilePath;
+    }
+  }
+
+  return dbConfig;
+}

packages/nc-secret-cli/src/nc-config/index.ts

@@ -0,0 +1,4 @@
+export * from './helpers';
+export * from './interfaces';
+export * from './constants';
+export * from './NcConfig';

packages/nc-secret-cli/src/nc-config/interfaces.ts

@@ -0,0 +1,39 @@
+import { DriverClient } from './constants';
+
+interface Connection {
+  driver?: DriverClient;
+  host?: string;
+  port?: number;
+  database?: string;
+  user?: string;
+  password?: string;
+  ssl?:
+    | boolean
+    | {
+        ca?: string;
+        cert?: string;
+        key?: string;
+        caFilePath?: string;
+        certFilePath?: string;
+        keyFilePath?: string;
+      };
+  filename?: string;
+}
+
+interface DbConfig {
+  client: DriverClient;
+  connection: Connection;
+  acquireConnectionTimeout?: number;
+  useNullAsDefault?: boolean;
+  pool?: {
+    min?: number;
+    max?: number;
+    idleTimeoutMillis?: number;
+  };
+  migrations?: {
+    directory?: string;
+    tableName?: string;
+  };
+}
+
+export { DriverClient, Connection, DbConfig };

packages/nc-secret-cli/tsconfig.json

@@ -0,0 +1,14 @@
+{
+    "compilerOptions": {
+      "rootDir": "src",
+      "outDir": "dist",
+      "strict": true,
+      "target": "es6",
+      "module": "commonjs",
+      "sourceMap": true,
+      "esModuleInterop": true,
+      "moduleResolution": "node",
+      "skipLibCheck": true,
+      "noImplicitAny": false
+    }
+  }

packages/nocodb/src/index.ts

@@ -3,3 +3,6 @@ import Noco from './Noco';
 export default Noco;
 
 export { Noco };
+export { SqlClientFactory } from './db/sql-client/lib/SqlClientFactory';
+export { MetaTable } from '~/utils/globals'
+export * from '~/utils/encryptDecrypt'