From ac7112963b3b24947dfb90007d29be8c9627e1f3 Mon Sep 17 00:00:00 2001 From: Salim B Date: Wed, 15 May 2024 23:50:08 +0200 Subject: [PATCH] feat: expose Litestream configuration --- .../020.environment-variables.md | 124 +++++++++--------- packages/nocodb/Dockerfile | 17 ++- packages/nocodb/docker/litestream.yml | 22 ++++ packages/nocodb/docker/start-litestream.sh | 14 +- packages/nocodb/litestream/Dockerfile | 22 ++-- 5 files changed, 117 insertions(+), 82 deletions(-) create mode 100644 packages/nocodb/docker/litestream.yml diff --git a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md index 8b66160ece..186a0f33cd 100644 --- a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md +++ b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md @@ -12,62 +12,68 @@ For production use-cases, it is **recommended** to configure - `NC_PUBLIC_URL`, - `NC_REDIS_URL` -| Variable | Comments | If absent | -|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| -| NC_DB | See our example database URLs [here](https://github.com/nocodb/nocodb#docker). | A local SQLite will be created in root folder if `NC_DB` is not provided | -| NC_DB_JSON | Can be used instead of `NC_DB` and value should be valid knex connection JSON | | -| NC_DB_JSON_FILE | Can be used instead of `NC_DB` and value should be a valid path to knex connection JSON | | -| DATABASE_URL | Can be used instead of `NC_DB` and value should be in JDBC URL format | | -| DATABASE_URL_FILE | Can be used instead of `DATABASE_URL` and value should be a valid path to file containing JDBC URL format. | | -| NC_AUTH_JWT_SECRET | JWT secret used for auth and storing other secrets | A random secret will be generated | -| PORT | For setting app running port | `8080` | -| DB_QUERY_LIMIT_DEFAULT | Pagination limit | 25 | -| DB_QUERY_LIMIT_GROUP_BY_GROUP | Group per page limit | 10 | -| DB_QUERY_LIMIT_GROUP_BY_RECORD | Record per group limit | 10 | -| DB_QUERY_LIMIT_MAX | Maximum allowed pagination limit | 1000 | -| DB_QUERY_LIMIT_MIN | Minimum allowed pagination limit | 1 | -| NC_TOOL_DIR | App directory to keep metadata and app related files | Defaults to current working directory. In docker maps to `/usr/app/data/` for mounting volume. | -| NC_PUBLIC_URL | Used for sending Email invitations | Best guess from http request params | -| NC_JWT_EXPIRES_IN | JWT token expiry time | `10h` | -| NC_CONNECT_TO_EXTERNAL_DB_DISABLED | Disable Project creation with external database | | -| NC_INVITE_ONLY_SIGNUP | Removed since version 0.99.0 and now it's recommended to use [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup). Allow users to signup only via invite URL, value should be any non-empty string. | | -| NUXT_PUBLIC_NC_BACKEND_URL | Custom Backend URL | ``http://localhost:8080`` will be used | -| NC_REQUEST_BODY_SIZE | Request body size [limit](https://expressjs.com/en/resources/middleware/body-parser.html#limit) | `1048576` | -| NC_EXPORT_MAX_TIMEOUT | After NC_EXPORT_MAX_TIMEOUT, CSV gets downloaded in batches | Default value 5000(in millisecond) will be used | -| NC_DISABLE_TELE | Disable telemetry | | -| NC_DASHBOARD_URL | Custom dashboard URL path | `/dashboard` | -| NC_GOOGLE_CLIENT_ID | Google client ID to enable Google authentication | | -| NC_GOOGLE_CLIENT_SECRET | Google client secret to enable Google authentication | | -| NC_MIGRATIONS_DISABLED | Disable NocoDB migration | | -| NC_MIN | If set to any non-empty string the default splash screen(initial welcome animation) and matrix screensaver will disable | | -| NC_SENTRY_DSN | For Sentry monitoring | | -| NC_REDIS_URL | Custom Redis URL. Example: `redis://:authpassword@127.0.0.1:6380/4` | Meta data will be stored in memory | -| NC_DISABLE_ERR_REPORT | Disable error reporting | | -| NC_DISABLE_CACHE | To be used only while debugging. On setting this to `true` - meta data be fetched from db instead of redis/cache. | `false` | -| AWS_ACCESS_KEY_ID | For Litestream - S3 access key id | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | -| AWS_SECRET_ACCESS_KEY | For Litestream - S3 secret access key | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | -| AWS_BUCKET | For Litestream - S3 bucket | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | -| AWS_BUCKET_PATH | For Litestream - S3 bucket path (like folder within S3 bucket) | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | -| NC_SMTP_FROM | For SMTP plugin - Email sender address | | -| NC_SMTP_HOST | For SMTP plugin - SMTP host value | | -| NC_SMTP_PORT | For SMTP plugin - SMTP port value | | -| NC_SMTP_USERNAME | For SMTP plugin (Optional) - SMTP username value for authentication | | -| NC_SMTP_PASSWORD | For SMTP plugin (Optional) - SMTP password value for authentication | | -| NC_SMTP_SECURE | For SMTP plugin (Optional) - To enable secure set value as `true` any other value treated as false | | -| NC_SMTP_IGNORE_TLS | For SMTP plugin (Optional) - To ignore tls set value as `true` any other value treated as false. For more info visit https://nodemailer.com/smtp/ | | -| NC_S3_BUCKET_NAME | For S3 storage plugin - AWS S3 bucket name | | -| NC_S3_REGION | For S3 storage plugin - AWS S3 region | | -| NC_S3_ACCESS_KEY | For S3 storage plugin - AWS access key credential for accessing resource | | -| NC_S3_ACCESS_SECRET | For S3 storage plugin - AWS access secret credential for accessing resource | | -| NC_ATTACHMENT_FIELD_SIZE | For setting the attachment field size(in Bytes) | Defaults to 20MB | -| NC_MAX_ATTACHMENTS_ALLOWED | Maximum Number of attachments per cell | | -| NC_ADMIN_EMAIL | For updating/creating super admin with provided email and password | | -| NC_ADMIN_PASSWORD | For updating/creating super admin with provided email and password. Your password should have at least 8 letters with one uppercase, one number and one special letter(Allowed special chars $&+,:;=?@#\|'.^*()%!_-" ) | | -| NODE_OPTIONS | For passing Node.js [options](https://nodejs.org/api/cli.html#node_optionsoptions) to instance | | -| NC_MINIMAL_DBS | Create a new SQLite file for each project. All the db files are stored in `nc_minimal_dbs` folder in current working directory. (This option restricts project creation on external sources) | | -| NC_DISABLE_AUDIT | Disable Audit Log | `false` | -| NC_AUTOMATION_LOG_LEVEL | Possible Values: `OFF`, `ERROR`, `ALL`. See [Webhooks](/automation/webhook/create-webhook#call-log) for details. | `OFF` | -| NC_SECURE_ATTACHMENTS | Allow accessing attachments only through presigned urls. To enable set value as `true` any other value treated as false. (⚠ this will make existing links inaccessible ⚠) | `false` | -| NC_ATTACHMENT_EXPIRE_SECONDS | How many seconds before expiring presigned attachment urls. (Attachments will expire in at least set seconds and at most 10mins after set time) | 7200 (2 hours) | -| NC_ALLOW_LOCAL_HOOKS | To enable set value as `true` any other value treated as false. (⚠ this will allow webhooks to call local links which can raise security issues ⚠) | `false` | -| NC_SANITIZE_COLUMN_NAME | Sanitize the column name during column creation. To enable set value as `true` any other value treated as false. | `true` | +| Variable | Description | If absent | +| -------- | ----------- | --------- | +| `NC_DB` | See our example database URLs [here](https://github.com/nocodb/nocodb#docker). | A local SQLite database will be created in root folder if `NC_DB` is not provided | +| `NC_DB_JSON` | Can be used instead of `NC_DB` and value should be valid knex connection JSON | | +| `NC_DB_JSON_FILE` | Can be used instead of `NC_DB` and value should be a valid path to knex connection JSON | | +| `DATABASE_URL` | Can be used instead of `NC_DB` and value should be in JDBC URL format | | +| `DATABASE_URL_FILE` | Can be used instead of `DATABASE_URL` and value should be a valid path to file containing JDBC URL format. | | +| `NC_AUTH_JWT_SECRET` | JWT secret used for auth and storing other secrets | A random secret will be generated | +| `PORT` | For setting app running port | `8080` | +| `DB_QUERY_LIMIT_DEFAULT` | Pagination limit | `25` | +| `DB_QUERY_LIMIT_GROUP_BY_GROUP` | Group per page limit | `10` | +| `DB_QUERY_LIMIT_GROUP_BY_RECORD` | Record per group limit | `10` | +| `DB_QUERY_LIMIT_MAX` | Maximum allowed pagination limit | `1000` | +| `DB_QUERY_LIMIT_MIN` | Minimum allowed pagination limit | `1` | +| `NC_TOOL_DIR` | App directory to keep metadata and app related files | Defaults to current working directory. In docker maps to `/usr/app/data/` for mounting volume. | +| `NC_PUBLIC_URL` | Used for sending Email invitations | Best guess from http request params | +| `NC_JWT_EXPIRES_IN` | JWT token expiry time | `10h` | +| `NC_CONNECT_TO_EXTERNAL_DB_DISABLED` | Disable Project creation with external database | | +| `NC_INVITE_ONLY_SIGNUP` | Removed since version 0.99.0 and now it's recommended to use [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup). Allow users to signup only via invite URL, value should be any non-empty string. | | +| `NUXT_PUBLIC_NC_BACKEND_URL` | Custom Backend URL | `http://localhost:8080` will be used | +| `NC_REQUEST_BODY_SIZE` | Request body size [limit](https://expressjs.com/en/resources/middleware/body-parser.html#limit) | `1048576` | +| `NC_EXPORT_MAX_TIMEOUT` | After `NC_EXPORT_MAX_TIMEOUT`, CSV gets downloaded in batches | Default value `5000` (in milliseconds) will be used | +| `NC_DISABLE_TELE` | Disable telemetry | | +| `NC_DASHBOARD_URL` | Custom dashboard URL path | `/dashboard` | +| `NC_GOOGLE_CLIENT_ID` | Google client ID to enable Google authentication | | +| `NC_GOOGLE_CLIENT_SECRET` | Google client secret to enable Google authentication | | +| `NC_MIGRATIONS_DISABLED` | Disable NocoDB migration | | +| `NC_MIN` | If set to any non-empty string the default splash screen (initial welcome animation) and matrix screensaver will disable | | +| `NC_SENTRY_DSN` | For Sentry monitoring | | +| `NC_REDIS_URL` | Custom Redis URL. Example: `redis://:authpassword@127.0.0.1:6380/4` | Meta data will be stored in memory | +| `NC_DISABLE_ERR_REPORT` | Disable error reporting | | +| `NC_DISABLE_CACHE` | To be used only while debugging. On setting this to `true` - meta data be fetched from db instead of redis/cache. | `false` | +| `NC_SMTP_FROM` | For SMTP plugin - Email sender address | | +| `NC_SMTP_HOST` | For SMTP plugin - SMTP host value | | +| `NC_SMTP_PORT` | For SMTP plugin - SMTP port value | | +| `NC_SMTP_USERNAME` | For SMTP plugin (Optional) - SMTP username value for authentication | | +| `NC_SMTP_PASSWORD` | For SMTP plugin (Optional) - SMTP password value for authentication | | +| `NC_SMTP_SECURE` | For SMTP plugin (Optional) - To enable secure set value as `true` any other value treated as false | | +| `NC_SMTP_IGNORE_TLS` | For SMTP plugin (Optional) - To ignore tls set value as `true` any other value treated as false. For more info visit https://nodemailer.com/smtp/ | | +| `NC_S3_BUCKET_NAME` | For S3 storage plugin - AWS S3 bucket name | | +| `NC_S3_REGION` | For S3 storage plugin - AWS S3 region | | +| `NC_S3_ACCESS_KEY` | For S3 storage plugin - AWS access key credential for accessing resource | | +| `NC_S3_ACCESS_SECRET` | For S3 storage plugin - AWS access secret credential for accessing resource | | +| `NC_ATTACHMENT_FIELD_SIZE` | For setting the attachment field size(in Bytes) | Defaults to 20MB | +| `NC_MAX_ATTACHMENTS_ALLOWED` | Maximum Number of attachments per cell | | +| `NC_ADMIN_EMAIL` | For updating/creating super admin with provided email and password | | +| `NC_ADMIN_PASSWORD` | For updating/creating super admin with provided email and password. Your password should have at least 8 letters with one uppercase, one number and one special letter. Allowed special characters include `$&+,:;=?@#\|'.^*()%!_-"`. ) | | +| `NODE_OPTIONS` | For passing Node.js [options](https://nodejs.org/api/cli.html#node_optionsoptions) to instance | | +| `NC_MINIMAL_DBS` | Create a new SQLite file for each project. All the db files are stored in `nc_minimal_dbs` folder in current working directory. (This option restricts project creation on external sources) | | +| `NC_DISABLE_AUDIT` | Disable Audit Log | `false` | +| `NC_AUTOMATION_LOG_LEVEL` | Possible Values: `OFF`, `ERROR`, `ALL`. See [Webhooks](/automation/webhook/create-webhook#call-log) for details. | `OFF` | +| `NC_SECURE_ATTACHMENTS` | Allow accessing attachments only through presigned urls. To enable set value as `true` any other value treated as false. (⚠ this will make existing links inaccessible ⚠) | `false` | +| `NC_ATTACHMENT_EXPIRE_SECONDS` | How many seconds before expiring presigned attachment urls. (Attachments will expire in at least set seconds and at most 10mins after set time) | 7200 (2 hours) | +| `NC_ALLOW_LOCAL_HOOKS` | To enable set value as `true` any other value treated as false. (⚠ this will allow webhooks to call local links which can raise security issues ⚠) | `false` | +| `NC_SANITIZE_COLUMN_NAME` | Sanitize the column name during column creation. To enable set value as `true` any other value treated as false. | `true` | +| `LITESTREAM_S3_ENDPOINT` | URL of an S3-compatible object storage service endpoint like `s3.eu-central-1.amazonaws.com`. | *Litestream replication is disabled if this variable is not set.* | +| `LITESTREAM_S3_BUCKET` | Name of the S3-compatible object storage bucket to store the Litestream replication in. | *Litestream replication is disabled if this variable is not set.* | +| `LITESTREAM_S3_PATH` | Directory path to use within the Litestream replication bucket. | Defaults to `nocodb`. | +| `LITESTREAM_S3_ACCESS_KEY_ID` | Litestream authentication key for the S3 replica. | *Litestream replication is disabled if this variable is not set.* | +| `LITESTREAM_S3_SECRET_ACCESS_KEY` | Litestream authentication key for the S3 replica. | *Litestream replication is disabled if this variable is not set.* | +| `LITESTREAM_S3_SKIP_VERIFY` | Whether to disable TLS verification. This is useful when testing against a local node such as MinIO and you are using self-signed certificates. | Defaults to `false`. | +| `LITESTREAM_RETENTION` | Amount of time Litestream snapshot and WAL files are kept. After the retention period, a new snapshot is created and the old one is removed. WAL files that exist before the oldest snapshot will also be removed. | Defaults to `1440h` (60 days). | +| `LITESTREAM_RETENTION_CHECK_INTERVAL` | Frequency in which Litestream will check if retention needs to be enforced. | Defaults to `72h` (3 days). | +| `LITESTREAM_SNAPSHOT_INTERVAL` | Frequency in which new Litestream snapshots are created. A higher frequency reduces the time to restore since newer snapshots will have fewer WAL frames to apply. Retention still applies to these snapshots. | Defaults to `24h` (1 day). | +| `LITESTREAM_SYNC_INTERVAL` | Frequency in which frames are pushed to the Litestream replica. Increasing this frequency can increase object storage costs significantly. | Defaults to `60s` (1 minute). | diff --git a/packages/nocodb/Dockerfile b/packages/nocodb/Dockerfile index 5a31b6f6ab..0b7e56eceb 100644 --- a/packages/nocodb/Dockerfile +++ b/packages/nocodb/Dockerfile @@ -52,17 +52,24 @@ RUN pnpm install --prod --shamefully-hoist \ FROM alpine:3.19 WORKDIR /usr/src/app -ENV NC_DOCKER 0.6 -ENV NODE_ENV production -ENV PORT 8080 -ENV NC_TOOL_DIR=/usr/app/data/ +ENV LITESTREAM_S3_SKIP_VERIFY=false \ + LITESTREAM_S3_PATH=nocodb \ + LITESTREAM_RETENTION=1440h \ + LITESTREAM_RETENTION_CHECK_INTERVAL=72h \ + LITESTREAM_SNAPSHOT_INTERVAL=24h \ + LITESTREAM_SYNC_INTERVAL=60s \ + NC_DOCKER=0.6 \ + NC_TOOL_DIR=/usr/app/data/ \ + NODE_ENV=production \ + PORT=8080 RUN apk --update --no-cache add \ nodejs \ dumb-init -# Copy litestream binary +# Copy litestream binary and config file COPY --from=lt-builder /usr/src/lt /usr/local/bin/litestream +COPY ./docker/litestream.yml /etc/litestream.yml # Copy production code & main entry file COPY --from=builder /usr/src/app/ /usr/src/app/ COPY --from=builder /usr/src/appEntry/ /usr/src/appEntry/ diff --git a/packages/nocodb/docker/litestream.yml b/packages/nocodb/docker/litestream.yml new file mode 100644 index 0000000000..4306b7b21b --- /dev/null +++ b/packages/nocodb/docker/litestream.yml @@ -0,0 +1,22 @@ +# Docs: https://litestream.io/reference/config/ + +dbs: + - path: ${NC_TOOL_DIR}noco.db + replicas: + - type: s3 + endpoint: ${LITESTREAM_S3_ENDPOINT} + force-path-style: true + skip-verify: ${LITESTREAM_S3_SKIP_VERIFY} + bucket: ${LITESTREAM_S3_BUCKET} + path: ${LITESTREAM_S3_PATH} + access-key-id: ${LITESTREAM_S3_ACCESS_KEY_ID} + secret-access-key: ${LITESTREAM_S3_SECRET_ACCESS_KEY} + retention: ${LITESTREAM_RETENTION} + retention-check-interval: ${LITESTREAM_RETENTION_CHECK_INTERVAL} + snapshot-interval: ${LITESTREAM_SNAPSHOT_INTERVAL} + sync-interval: ${LITESTREAM_SYNC_INTERVAL} + # age: + # identities: + # - ${LITESTREAM_AGE_SECRET_KEY} + # recipients: + # - ${LITESTREAM_AGE_PUBLIC_KEY} diff --git a/packages/nocodb/docker/start-litestream.sh b/packages/nocodb/docker/start-litestream.sh index db5987faef..da66e5480b 100644 --- a/packages/nocodb/docker/start-litestream.sh +++ b/packages/nocodb/docker/start-litestream.sh @@ -6,21 +6,21 @@ if [ -n "${NC_TOOL_DIR}" ]; then mkdir -p "$NC_TOOL_DIR" fi -if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ] && [ -n "${AWS_BUCKET}" ] && [ -n "${AWS_BUCKET_PATH}" ]; then +if [ -n "${LITESTREAM_S3_ENDPOINT}" ] && [ -n "${LITESTREAM_S3_BUCKET}" ] && [ -n "${LITESTREAM_ACCESS_KEY_ID}" ] && [ -n "${LITESTREAM_SECRET_ACCESS_KEY}" ] ; then - if [ -f "${NC_TOOL_DIR}noco.db" ] - then + if [ -f "${NC_TOOL_DIR}noco.db" ] ; then rm "${NC_TOOL_DIR}noco.db" rm "${NC_TOOL_DIR}noco.db-shm" rm "${NC_TOOL_DIR}noco.db-wal" fi - litestream restore -o "${NC_TOOL_DIR}noco.db" "s3://$AWS_BUCKET/$AWS_BUCKET_PATH" - if [ ! -f "${NC_TOOL_DIR}noco.db" ] - then + litestream restore "${NC_TOOL_DIR}noco.db" + + if [ ! -f "${NC_TOOL_DIR}noco.db" ] ; then touch "${NC_TOOL_DIR}noco.db" fi - litestream replicate "${NC_TOOL_DIR}noco.db" "s3://$AWS_BUCKET/$AWS_BUCKET_PATH" & + + litestream replicate & fi node docker/main.js diff --git a/packages/nocodb/litestream/Dockerfile b/packages/nocodb/litestream/Dockerfile index 116dc8ea1f..ec45e80e7f 100644 --- a/packages/nocodb/litestream/Dockerfile +++ b/packages/nocodb/litestream/Dockerfile @@ -38,12 +38,6 @@ RUN chmod +x /usr/src/appEntry/start.sh FROM alpine:3.19 -#ENV AWS_ACCESS_KEY_ID= -#ENV AWS_SECRET_ACCESS_KEY= -#ENV AWS_BUCKET= - - - #WORKDIR /usr/src/ # ## Install go lang @@ -68,10 +62,15 @@ FROM alpine:3.19 WORKDIR /usr/src/app -ENV NC_DOCKER 0.6 -ENV PORT 8080 -ENV NC_TOOL_DIR=/usr/app/data/ - +ENV LITESTREAM_S3_SKIP_VERIFY=false \ + LITESTREAM_S3_PATH=nocodb \ + LITESTREAM_RETENTION=1440h \ + LITESTREAM_RETENTION_CHECK_INTERVAL=72h \ + LITESTREAM_SNAPSHOT_INTERVAL=24h \ + LITESTREAM_SYNC_INTERVAL=60s \ + NC_DOCKER=0.6 \ + NC_TOOL_DIR=/usr/app/data/ \ + PORT=8080 # Copy application dependency manifests to the container image. # A wildcard is used to ensure both package.json AND package-lock.json are copied. @@ -84,8 +83,9 @@ RUN apk --update --no-cache add \ nodejs \ tar -# Copy litestream binary +# Copy litestream binary and config file COPY --from=lt /usr/src/lt /usr/local/bin/litestream +COPY ./docker/litestream.yml /etc/litestream.yml # Copy production code & main entry file COPY --from=builder /usr/src/app/ /usr/src/app/ COPY --from=builder /usr/src/appEntry/ /usr/src/appEntry/