From 0ea94d65aa0eac527c8a2a68999357483781f482 Mon Sep 17 00:00:00 2001 From: Salim B Date: Wed, 22 May 2024 06:48:26 +0200 Subject: [PATCH] feat: support Litestream age encryption (#8526) * feat: support Litestream age encryption * fix: correct Litestream S3 env vars to match doc * fix: add backwards compatibility for renamed env vars --- .../020.environment-variables.md | 4 +++ packages/nocodb/Dockerfile | 5 +-- packages/nocodb/docker/start-litestream.sh | 31 +++++++++++++++++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md index c0349a455d..40ddb5ea94 100644 --- a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md +++ b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md @@ -78,3 +78,7 @@ For production use cases, it is **recommended** to set at least: | `LITESTREAM_RETENTION_CHECK_INTERVAL` | Frequency in which Litestream will check if retention needs to be enforced. | Defaults to `72h` (3 days). | | `LITESTREAM_SNAPSHOT_INTERVAL` | Frequency in which new Litestream snapshots are created. A higher frequency reduces the time to restore since newer snapshots will have fewer WAL frames to apply. Retention still applies to these snapshots. | Defaults to `24h` (1 day). | | `LITESTREAM_SYNC_INTERVAL` | Frequency in which frames are pushed to the Litestream replica. Increasing this frequency can increase object storage costs significantly. | Defaults to `60s` (1 minute). | +| `LITESTREAM_AGE_PUBLIC_KEY` | [age](https://age-encryption.org/) public key generated by `age-keygen` (`age1...`) or SSH public key (`ssh-ed25519 AAAA...`, `ssh-rsa AAAA...`) used to encrypt the Litestream replication for. Refer to the relevant [Litestream documentation](https://litestream.io/reference/config/#encryption) for details. | *Litestream replication is unencrypted if this variable is not set.* | +| `LITESTREAM_AGE_SECRET_KEY` | [age](https://age-encryption.org/) secret key (`AGE-SECRET-KEY-1...`) used to encrypt the Litestream replication with. Refer to the relevant [Litestream documentation](https://litestream.io/reference/config/#encryption) for details. | *Litestream replication is unencrypted if this variable is not set.* | +| `AWS_ACCESS_KEY_ID` | ***Deprecated***. Please use `LITESTREAM_S3_ACCESS_KEY_ID` instead. | | +| `AWS_SECRET_ACCESS_KEY` | ***Deprecated***. Please use `LITESTREAM_S3_SECRET_ACCESS_KEY` instead. | | diff --git a/packages/nocodb/Dockerfile b/packages/nocodb/Dockerfile index 30831def3f..c027420d92 100644 --- a/packages/nocodb/Dockerfile +++ b/packages/nocodb/Dockerfile @@ -62,8 +62,9 @@ ENV LITESTREAM_S3_SKIP_VERIFY=false \ PORT=8080 RUN apk add --update --no-cache \ - nodejs \ - dumb-init + dasel \ + dumb-init \ + nodejs # Copy litestream binary and config file COPY --link --from=lt-builder /usr/src/lt /usr/local/bin/litestream diff --git a/packages/nocodb/docker/start-litestream.sh b/packages/nocodb/docker/start-litestream.sh index 8d75d5a4ef..79c2ac2e00 100644 --- a/packages/nocodb/docker/start-litestream.sh +++ b/packages/nocodb/docker/start-litestream.sh @@ -4,6 +4,14 @@ if [ ! -d "${NC_TOOL_DIR}" ] ; then mkdir -p "$NC_TOOL_DIR" fi +# ensure backwards compatibility of renamed env vars +if [ -z "${LITESTREAM_S3_ACCESS_KEY_ID}" ] && [ -n "${AWS_ACCESS_KEY_ID}" ] ; then + export LITESTREAM_S3_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" +fi +if [ -z "${LITESTREAM_S3_SECRET_ACCESS_KEY}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ] ; then + export LITESTREAM_S3_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" +fi + use_litestream() { [ -z "${NC_DB}" ] \ && [ -z "${NC_DB_JSON}" ] \ @@ -13,25 +21,44 @@ use_litestream() { && [ -z "${NC_MINIMAL_DBS}" ] \ && [ -n "${LITESTREAM_S3_ENDPOINT}" ] \ && [ -n "${LITESTREAM_S3_BUCKET}" ] \ - && [ -n "${LITESTREAM_ACCESS_KEY_ID}" ] \ - && [ -n "${LITESTREAM_SECRET_ACCESS_KEY}" ] + && [ -n "${LITESTREAM_S3_ACCESS_KEY_ID}" ] \ + && [ -n "${LITESTREAM_S3_SECRET_ACCESS_KEY}" ] } if use_litestream ; then + # enable age encryption in Litestream config if indicated + LITESTREAM_CONFIG_PATH='/etc/litestream.yml' + + if [ -n "${LITESTREAM_AGE_PUBLIC_KEY}" ] \ + && [ -n "${LITESTREAM_AGE_SECRET_KEY}" ] \ + && ! dasel --file "${LITESTREAM_CONFIG_PATH}" --read yaml 'dbs.first().replicas.first().age' > /dev/null 2>&1 ; then + # shellcheck disable=SC2016 + dasel put --file "${LITESTREAM_CONFIG_PATH}" \ + --read yaml \ + --type json \ + --value '{ "identities": [ "${LITESTREAM_AGE_SECRET_KEY}" ], "recipients": [ "${LITESTREAM_AGE_PUBLIC_KEY}" ] }' \ + --selector 'dbs.first().replicas.first().age' + fi + + # remove any possible local DB leftovers if [ -f "${NC_TOOL_DIR}noco.db" ] ; then rm "${NC_TOOL_DIR}noco.db" rm -f "${NC_TOOL_DIR}noco.db-shm" rm -f "${NC_TOOL_DIR}noco.db-wal" fi + # restore DB from Litestream replica litestream restore "${NC_TOOL_DIR}noco.db" + # create empty DB file if no Litestream replica exists if [ ! -f "${NC_TOOL_DIR}noco.db" ] ; then touch "${NC_TOOL_DIR}noco.db" fi + # start Litestream replication litestream replicate & fi +# start NocoDB node docker/main.js