From 4179eb4ed21ad742834a9e7522ef71cd8bc93c6c Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sun, 6 Oct 2024 07:21:33 +0000 Subject: [PATCH] feat: cli for updating secret --- .gitignore | 2 + .../nc-secret-cli/src/core/SecretManager.ts | 112 ++++++ packages/nc-secret-cli/src/index.ts | 43 +++ .../nc-secret-cli/src/nc-config/NcConfig.ts | 95 +++++ .../nc-secret-cli/src/nc-config/constants.ts | 86 +++++ .../nc-secret-cli/src/nc-config/helpers.ts | 326 ++++++++++++++++++ packages/nc-secret-cli/src/nc-config/index.ts | 4 + .../nc-secret-cli/src/nc-config/interfaces.ts | 39 +++ packages/nc-secret-cli/tsconfig.json | 14 + packages/nocodb/src/index.ts | 3 + 10 files changed, 724 insertions(+) create mode 100644 packages/nc-secret-cli/src/core/SecretManager.ts create mode 100644 packages/nc-secret-cli/src/index.ts create mode 100644 packages/nc-secret-cli/src/nc-config/NcConfig.ts create mode 100644 packages/nc-secret-cli/src/nc-config/constants.ts create mode 100644 packages/nc-secret-cli/src/nc-config/helpers.ts create mode 100644 packages/nc-secret-cli/src/nc-config/index.ts create mode 100644 packages/nc-secret-cli/src/nc-config/interfaces.ts create mode 100644 packages/nc-secret-cli/tsconfig.json diff --git a/.gitignore b/.gitignore index 58c361aa98..4be0aa478b 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,5 @@ test_noco.db httpbin .run/test-debug.run.xml +/packages/nc-secret-cli/dist/index.js +/packages/nc-secret-cli/dist/index.js.map diff --git a/packages/nc-secret-cli/src/core/SecretManager.ts b/packages/nc-secret-cli/src/core/SecretManager.ts new file mode 100644 index 0000000000..3c1cd340e8 --- /dev/null +++ b/packages/nc-secret-cli/src/core/SecretManager.ts @@ -0,0 +1,112 @@ +import { SqlClientFactory, MetaTable, decryptPropIfRequired, encryptPropIfRequired } from 'nocodb'; + +export class SecretManager { + + private sqlClient; + + constructor(private oldSecret: string, private newSecret: string, private config: any) { + this.sqlClient = SqlClientFactory.create(this.config.meta.db); + } + + + // validate config by checking if database config is valid + async validateConfig() { + // use the sqlClientFactory to create a new sql client and then use testConnection to test the connection + const isValid = await this.sqlClient.testConnection(); + if (!isValid) { + throw new Error('Invalid database configuration'); + } + } + + + async validateAndExtract() { + const sources = await this.sqlClient.knex(MetaTable.SOURCES).where(qb => { + qb.where('is_meta', false).orWhere('is_meta', null) + }); + + const integrations = await this.sqlClient.knex(MetaTable.INTEGRATIONS).where(qb => { + qb.where('is_meta', false).orWhere('is_meta', null) + }); + + const sourcesToUpdate: Record[] = []; + const integrationsToUpdate: Record[] = []; + + + let isValid = false; + for (const source of sources) { + try { + const decrypted = decryptPropIfRequired({ + data: source, + secret: this.oldSecret, + prop: 'config' + }); + isValid = true; + sourcesToUpdate.push({ ...source, config: decrypted }); + } catch (e) { + console.log(e); + } + } + + for (const integration of integrations) { + try { + const decrypted = decryptPropIfRequired({ + data: integration, + secret: this.oldSecret, + prop: 'config' + }); + isValid = true; + integrationsToUpdate.push({ ...integration, config: decrypted }); + } catch (e) { + console.log(e); + } + } + + // if all of the decyptions are failed then throw an error + if (!isValid) { + throw new Error('Invalid old secret or no sources/integrations found'); + } + + + return { sourcesToUpdate, integrationsToUpdate }; + } + + + async updateSecret( + sourcesToUpdate: Record[], + integrationsToUpdate: Record[] + ) { + // start transaction + const transaction = await this.sqlClient.transaction(); + + try { + // update sources + for (const source of sourcesToUpdate) { + await transaction(MetaTable.SOURCES).update({ + config: encryptPropIfRequired({ + data: source, + secret: this.newSecret, + prop: 'config' + }) + }).where('id', source.id); + } + + // update integrations + for (const integration of integrationsToUpdate) { + await transaction(MetaTable.INTEGRATIONS).update({ + config: encryptPropIfRequired({ + data: integration, + secret: this.newSecret, + prop: 'config' + }) + }).where('id', integration.id); + } + + await transaction.commit(); + + } catch (e) { + console.log(e); + await transaction.rollback(); + throw e; + } + } +} diff --git a/packages/nc-secret-cli/src/index.ts b/packages/nc-secret-cli/src/index.ts new file mode 100644 index 0000000000..7961faca02 --- /dev/null +++ b/packages/nc-secret-cli/src/index.ts @@ -0,0 +1,43 @@ +import figlet from "figlet"; + +console.log(figlet.textSync("Nocodb Secret CLI")); + +import { Command } from 'commander'; +import { getNocoConfig } from "./nc-config"; +import { SecretManager } from "./core/SecretManager"; + +const program = new Command(); + +program + .version('1.0.0') + .description('NocoDB Secret CLI') + .arguments(' ') + .action(async (key, value) => { + + const config = await getNocoConfig(); + + if (!key || !value) { + console.error('Error: Both key and value are required.'); + program.help(); + } else { + const secretManager = new SecretManager(key, value, config); + + // validate meta db config which is resolved from env variables + await secretManager.validateConfig(); + + // validate old secret + const { sourcesToUpdate, integrationsToUpdate } = await secretManager.validateAndExtract(); + + + // update sources and integrations + await secretManager.updateSecret(sourcesToUpdate, integrationsToUpdate); + + } + }); + + + +// Add error handling +program.exitOverride(); + +program.parse(process.argv); diff --git a/packages/nc-secret-cli/src/nc-config/NcConfig.ts b/packages/nc-secret-cli/src/nc-config/NcConfig.ts new file mode 100644 index 0000000000..63f031d48f --- /dev/null +++ b/packages/nc-secret-cli/src/nc-config/NcConfig.ts @@ -0,0 +1,95 @@ +import * as path from 'path'; +import fs from 'fs'; +import { promisify } from 'util'; +import { getToolDir, metaUrlToDbConfig } from './helpers'; +import { DriverClient } from './interfaces'; +import type { DbConfig } from './interfaces'; +import { SqlClientFactory } from 'nocodb'; + +export class NcConfig { + meta: { + db: DbConfig; + } = { + db: { + client: DriverClient.SQLITE, + connection: { + filename: 'noco.db', + }, + }, + }; + + toolDir: string; + + credentialSecret?: string; + + private constructor() { + this.toolDir = getToolDir(); + } + + public static async create(param: { + meta: { + metaUrl?: string; + metaJson?: string; + metaJsonFile?: string; + }; + secret?: string; + credentialSecret?: string; + }): Promise { + const { meta, secret } = + param; + + const ncConfig = new NcConfig(); + + + ncConfig.credentialSecret = param.credentialSecret; + + + if (ncConfig.meta?.db?.connection?.filename) { + ncConfig.meta.db.connection.filename = path.join( + ncConfig.toolDir, + ncConfig.meta.db.connection.filename, + ); + } + + if (meta?.metaUrl) { + ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl); + } else if (meta?.metaJson) { + ncConfig.meta.db = JSON.parse(meta.metaJson); + } else if (meta?.metaJsonFile) { + if (!(await promisify(fs.exists)(meta.metaJsonFile))) { + throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`); + } + const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, { + encoding: 'utf8', + }); + ncConfig.meta.db = JSON.parse(fileContent); + } + + + return ncConfig; + } + + public static async createByEnv(): Promise { + return NcConfig.create({ + meta: { + metaUrl: process.env.NC_DB, + metaJson: process.env.NC_DB_JSON, + metaJsonFile: process.env.NC_DB_JSON_FILE, + }, + secret: process.env.NC_AUTH_JWT_SECRET, + credentialSecret: process.env.NC_KEY_CREDENTIAL_ENCRYPT, + }); + } +} + +export const getNocoConfig = () =>{ + return NcConfig.create({ + meta: { + metaUrl: process.env.NC_DB, + metaJson: process.env.NC_DB_JSON, + metaJsonFile: process.env.NC_DB_JSON_FILE, + }, + secret: process.env.NC_AUTH_JWT_SECRET, + credentialSecret: process.env.NC_KEY_CREDENTIAL_ENCRYPT, + }); +} diff --git a/packages/nc-secret-cli/src/nc-config/constants.ts b/packages/nc-secret-cli/src/nc-config/constants.ts new file mode 100644 index 0000000000..5a2fe8adb4 --- /dev/null +++ b/packages/nc-secret-cli/src/nc-config/constants.ts @@ -0,0 +1,86 @@ +export const driverClientMapping = { + mysql: 'mysql2', + mariadb: 'mysql2', + postgres: 'pg', + postgresql: 'pg', + sqlite: 'sqlite3', + mssql: 'mssql', +}; + +export const defaultClientPortMapping = { + mysql: 3306, + mysql2: 3306, + postgres: 5432, + pg: 5432, + mssql: 1433, +}; + +export const defaultConnectionConfig: any = { + // https://github.com/knex/knex/issues/97 + // timezone: process.env.NC_TIMEZONE || 'UTC', + dateStrings: true, +}; + +// default knex options +export const defaultConnectionOptions = { + pool: { + min: 0, + max: 10, + }, +}; + +export const avoidSSL = [ + 'localhost', + '127.0.0.1', + 'host.docker.internal', + '172.17.0.1', +]; + +export const knownQueryParams = [ + { + parameter: 'database', + aliases: ['d', 'db'], + }, + { + parameter: 'password', + aliases: ['p'], + }, + { + parameter: 'user', + aliases: ['u'], + }, + { + parameter: 'title', + aliases: ['t'], + }, + { + parameter: 'keyFilePath', + aliases: [], + }, + { + parameter: 'certFilePath', + aliases: [], + }, + { + parameter: 'caFilePath', + aliases: [], + }, + { + parameter: 'ssl', + aliases: [], + }, + { + parameter: 'options', + aliases: ['opt', 'opts'], + }, +]; + +export enum DriverClient { + MYSQL = 'mysql2', + MYSQL_LEGACY = 'mysql', + MSSQL = 'mssql', + PG = 'pg', + SQLITE = 'sqlite3', + SNOWFLAKE = 'snowflake', + DATABRICKS = 'databricks', +} diff --git a/packages/nc-secret-cli/src/nc-config/helpers.ts b/packages/nc-secret-cli/src/nc-config/helpers.ts new file mode 100644 index 0000000000..37e893f0a2 --- /dev/null +++ b/packages/nc-secret-cli/src/nc-config/helpers.ts @@ -0,0 +1,326 @@ +import fs from 'fs'; +import { URL } from 'url'; +import { promisify } from 'util'; +import parseDbUrl from 'parse-database-url'; +import { + avoidSSL, + defaultClientPortMapping, + defaultConnectionConfig, + defaultConnectionOptions, + driverClientMapping, + knownQueryParams, +} from './constants'; +import { DriverClient } from './interfaces'; +import type { Connection, DbConfig } from './interfaces'; + +export async function prepareEnv() { + if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { + const database_url = await promisify(fs.readFile)( + (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) as string, + 'utf-8', + ); + process.env.NC_DB = jdbcToXcUrl(database_url); + } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { + process.env.NC_DB = jdbcToXcUrl( + (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) as string, + ); + } +} + +export function getToolDir() { + return process.env.NC_TOOL_DIR || process.cwd(); +} + +export function jdbcToXcConfig(url: string): DbConfig { + // drop the jdbc prefix + url.replace(/^jdbc:/, ''); + + const config = parseDbUrl(url); + + const parsedConfig: Connection = {}; + + for (const [key, value] of Object.entries(config)) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedConfig[fnd.parameter] = value; + } else { + parsedConfig[key] = value; + } + } + + if (!parsedConfig?.port) { + parsedConfig.port = + defaultClientPortMapping[ + driverClientMapping[parsedConfig.driver as DriverClient] || parsedConfig.driver + ]; + } + + const { driver, ...connectionConfig } = parsedConfig; + + const client = driverClientMapping[driver as DriverClient] || driver; + + if ( + client === 'pg' && + !connectionConfig?.ssl && + !avoidSSL.includes(connectionConfig.host as string) + ) { + connectionConfig.ssl = true; + } + + return { + client: client, + connection: { + ...connectionConfig, + }, + } as DbConfig; +} + +export function jdbcToXcUrl(url: string): string { + // drop the jdbc prefix + url.replace(/^jdbc:/, ''); + + const config = parseDbUrl(url); + + const parsedConfig: Connection = {}; + + for (const [key, value] of Object.entries(config)) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedConfig[fnd.parameter] = value; + } else { + parsedConfig[key] = value; + } + } + + if (!parsedConfig?.port) { + parsedConfig.port = + defaultClientPortMapping[ + driverClientMapping[parsedConfig.driver as DriverClient] || parsedConfig.driver + ]; + } + + const { driver, host, port, database, user, password, ...extra } = + parsedConfig; + + const extraParams: string[] = []; + + for (const [key, value] of Object.entries(extra)) { + extraParams.push(`${key}=${encodeURIComponent(String(value))}`); + } + + const res = `${driverClientMapping[driver as DriverClient] || driver}://${host}${ + port ? `:${port}` : '' + }?${user ? `u=${encodeURIComponent(user)}&` : ''}${ + password ? `p=${encodeURIComponent(password)}&` : '' + }${database ? `d=${encodeURIComponent(database)}&` : ''}${extraParams.join( + '&', + )}`; + + return res; +} + +export function xcUrlToDbConfig( + urlString: string, + key = '', + type?: string, +): DbConfig { + const url = new URL(urlString); + + let dbConfig: DbConfig; + + if (url.protocol.startsWith('sqlite3')) { + dbConfig = { + client: 'sqlite3', + connection: { + client: 'sqlite3', + connection: { + filename: + url.searchParams.get('d') || url.searchParams.get('database'), + }, + database: url.searchParams.get('d') || url.searchParams.get('database'), + }, + } as any; + } else { + const parsedQuery = {}; + for (const [key, value] of url.searchParams.entries()) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedQuery[fnd.parameter] = value; + } else { + parsedQuery[key] = value; + } + } + + dbConfig = { + client: url.protocol.replace(':', '') as DriverClient, + connection: { + ...parsedQuery, + host: url.hostname, + port: +url.port, + }, + acquireConnectionTimeout: 600000, + }; + + if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { + dbConfig.connection.ssl = true; + } + + if ( + url.searchParams.get('keyFilePath') && + url.searchParams.get('certFilePath') && + url.searchParams.get('caFilePath') + ) { + dbConfig.connection.ssl = { + keyFilePath: url.searchParams.get('keyFilePath') as string, + certFilePath: url.searchParams.get('certFilePath') as string, + caFilePath: url.searchParams.get('caFilePath') as string, + }; + } + } + + /* TODO check if this is needed + if (config && !config.title) { + config.title = + url.searchParams.get('t') || + url.searchParams.get('title') || + this.generateRandomTitle(); + } + */ + + Object.assign(dbConfig, { + meta: { + tn: 'nc_evolutions', + allSchemas: + !!url.searchParams.get('allSchemas') || + !(url.searchParams.get('d') || url.searchParams.get('database')), + api: { + prefix: url.searchParams.get('apiPrefix') || '', + swagger: true, + type: + type || + ((url.searchParams.get('api') || url.searchParams.get('a')) as any) || + 'rest', + }, + dbAlias: url.searchParams.get('dbAlias') || `db${key}`, + metaTables: 'db', + migrations: { + disabled: false, + name: 'nc_evolutions', + }, + }, + }); + + return dbConfig; +} + +export async function metaUrlToDbConfig(urlString): Promise { + const url = new URL(urlString); + + let dbConfig: DbConfig; + + if (url.protocol.startsWith('sqlite3')) { + const db = url.searchParams.get('d') || url.searchParams.get('database'); + dbConfig = { + client: DriverClient.SQLITE, + connection: { + filename: db as string, + }, + ...(db === ':memory:' + ? { + pool: { + min: 1, + max: 1, + // disposeTimeout: 360000*1000, + idleTimeoutMillis: 360000 * 1000, + }, + } + : {}), + }; + } else { + const parsedQuery = {}; + for (const [key, value] of url.searchParams.entries()) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedQuery[fnd.parameter] = value; + } else { + parsedQuery[key] = value; + } + } + + dbConfig = { + client: url.protocol.replace(':', '') as DriverClient, + connection: { + ...defaultConnectionConfig, + ...parsedQuery, + host: url.hostname, + port: +url.port, + }, + acquireConnectionTimeout: 600000, + ...defaultConnectionOptions, + ...(url.searchParams.has('search_path') + ? { + searchPath: url.searchParams.get('search_path')?.split(','), + } + : {}), + }; + + if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { + dbConfig.connection.ssl = true; + } + } + + url.searchParams.forEach((_value, key) => { + let value: any = _value; + if (value === 'true') { + value = true; + } else if (value === 'false') { + value = false; + } else if (/^\d+$/.test(value)) { + value = +value; + } + // todo: implement config read from JSON file or JSON env val read + if ( + !['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes( + key, + ) + ) { + key.split('.').reduce((obj, k, i, arr) => { + return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); + }, dbConfig); + } + }); + + if ( + dbConfig?.connection?.ssl && + typeof dbConfig?.connection?.ssl === 'object' + ) { + if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { + dbConfig.connection.ssl.ca = ( + await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) + ).toString(); + delete dbConfig.connection.ssl.caFilePath; + } + if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { + dbConfig.connection.ssl.key = ( + await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) + ).toString(); + delete dbConfig.connection.ssl.keyFilePath; + } + if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) { + dbConfig.connection.ssl.cert = ( + await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) + ).toString(); + delete dbConfig.connection.ssl.certFilePath; + } + } + + return dbConfig; +} diff --git a/packages/nc-secret-cli/src/nc-config/index.ts b/packages/nc-secret-cli/src/nc-config/index.ts new file mode 100644 index 0000000000..e70aeceb43 --- /dev/null +++ b/packages/nc-secret-cli/src/nc-config/index.ts @@ -0,0 +1,4 @@ +export * from './helpers'; +export * from './interfaces'; +export * from './constants'; +export * from './NcConfig'; diff --git a/packages/nc-secret-cli/src/nc-config/interfaces.ts b/packages/nc-secret-cli/src/nc-config/interfaces.ts new file mode 100644 index 0000000000..afd296d045 --- /dev/null +++ b/packages/nc-secret-cli/src/nc-config/interfaces.ts @@ -0,0 +1,39 @@ +import { DriverClient } from './constants'; + +interface Connection { + driver?: DriverClient; + host?: string; + port?: number; + database?: string; + user?: string; + password?: string; + ssl?: + | boolean + | { + ca?: string; + cert?: string; + key?: string; + caFilePath?: string; + certFilePath?: string; + keyFilePath?: string; + }; + filename?: string; +} + +interface DbConfig { + client: DriverClient; + connection: Connection; + acquireConnectionTimeout?: number; + useNullAsDefault?: boolean; + pool?: { + min?: number; + max?: number; + idleTimeoutMillis?: number; + }; + migrations?: { + directory?: string; + tableName?: string; + }; +} + +export { DriverClient, Connection, DbConfig }; diff --git a/packages/nc-secret-cli/tsconfig.json b/packages/nc-secret-cli/tsconfig.json new file mode 100644 index 0000000000..d07540206b --- /dev/null +++ b/packages/nc-secret-cli/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "strict": true, + "target": "es6", + "module": "commonjs", + "sourceMap": true, + "esModuleInterop": true, + "moduleResolution": "node", + "skipLibCheck": true, + "noImplicitAny": false + } + } \ No newline at end of file diff --git a/packages/nocodb/src/index.ts b/packages/nocodb/src/index.ts index ca9b3f35ad..c4ca202869 100644 --- a/packages/nocodb/src/index.ts +++ b/packages/nocodb/src/index.ts @@ -3,3 +3,6 @@ import Noco from './Noco'; export default Noco; export { Noco }; +export { SqlClientFactory } from './db/sql-client/lib/SqlClientFactory'; +export { MetaTable } from '~/utils/globals' +export * from '~/utils/encryptDecrypt' \ No newline at end of file