Browse Source

feat: expose Litestream configuration

pull/8494/head
Salim B 2 months ago
parent
commit
ac7112963b
No known key found for this signature in database
GPG Key ID: C7657487017FE6D5
  1. 124
      packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md
  2. 17
      packages/nocodb/Dockerfile
  3. 22
      packages/nocodb/docker/litestream.yml
  4. 14
      packages/nocodb/docker/start-litestream.sh
  5. 22
      packages/nocodb/litestream/Dockerfile

124
packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md

@ -12,62 +12,68 @@ For production use-cases, it is **recommended** to configure
- `NC_PUBLIC_URL`,
- `NC_REDIS_URL`
| Variable | Comments | If absent |
|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------|
| NC_DB | See our example database URLs [here](https://github.com/nocodb/nocodb#docker). | A local SQLite will be created in root folder if `NC_DB` is not provided |
| NC_DB_JSON | Can be used instead of `NC_DB` and value should be valid knex connection JSON | |
| NC_DB_JSON_FILE | Can be used instead of `NC_DB` and value should be a valid path to knex connection JSON | |
| DATABASE_URL | Can be used instead of `NC_DB` and value should be in JDBC URL format | |
| DATABASE_URL_FILE | Can be used instead of `DATABASE_URL` and value should be a valid path to file containing JDBC URL format. | |
| NC_AUTH_JWT_SECRET | JWT secret used for auth and storing other secrets | A random secret will be generated |
| PORT | For setting app running port | `8080` |
| DB_QUERY_LIMIT_DEFAULT | Pagination limit | 25 |
| DB_QUERY_LIMIT_GROUP_BY_GROUP | Group per page limit | 10 |
| DB_QUERY_LIMIT_GROUP_BY_RECORD | Record per group limit | 10 |
| DB_QUERY_LIMIT_MAX | Maximum allowed pagination limit | 1000 |
| DB_QUERY_LIMIT_MIN | Minimum allowed pagination limit | 1 |
| NC_TOOL_DIR | App directory to keep metadata and app related files | Defaults to current working directory. In docker maps to `/usr/app/data/` for mounting volume. |
| NC_PUBLIC_URL | Used for sending Email invitations | Best guess from http request params |
| NC_JWT_EXPIRES_IN | JWT token expiry time | `10h` |
| NC_CONNECT_TO_EXTERNAL_DB_DISABLED | Disable Project creation with external database | |
| NC_INVITE_ONLY_SIGNUP | Removed since version 0.99.0 and now it's recommended to use [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup). Allow users to signup only via invite URL, value should be any non-empty string. | |
| NUXT_PUBLIC_NC_BACKEND_URL | Custom Backend URL | ``http://localhost:8080`` will be used |
| NC_REQUEST_BODY_SIZE | Request body size [limit](https://expressjs.com/en/resources/middleware/body-parser.html#limit) | `1048576` |
| NC_EXPORT_MAX_TIMEOUT | After NC_EXPORT_MAX_TIMEOUT, CSV gets downloaded in batches | Default value 5000(in millisecond) will be used |
| NC_DISABLE_TELE | Disable telemetry | |
| NC_DASHBOARD_URL | Custom dashboard URL path | `/dashboard` |
| NC_GOOGLE_CLIENT_ID | Google client ID to enable Google authentication | |
| NC_GOOGLE_CLIENT_SECRET | Google client secret to enable Google authentication | |
| NC_MIGRATIONS_DISABLED | Disable NocoDB migration | |
| NC_MIN | If set to any non-empty string the default splash screen(initial welcome animation) and matrix screensaver will disable | |
| NC_SENTRY_DSN | For Sentry monitoring | |
| NC_REDIS_URL | Custom Redis URL. Example: `redis://:authpassword@127.0.0.1:6380/4` | Meta data will be stored in memory |
| NC_DISABLE_ERR_REPORT | Disable error reporting | |
| NC_DISABLE_CACHE | To be used only while debugging. On setting this to `true` - meta data be fetched from db instead of redis/cache. | `false` |
| AWS_ACCESS_KEY_ID | For Litestream - S3 access key id | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 |
| AWS_SECRET_ACCESS_KEY | For Litestream - S3 secret access key | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 |
| AWS_BUCKET | For Litestream - S3 bucket | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 |
| AWS_BUCKET_PATH | For Litestream - S3 bucket path (like folder within S3 bucket) | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 |
| NC_SMTP_FROM | For SMTP plugin - Email sender address | |
| NC_SMTP_HOST | For SMTP plugin - SMTP host value | |
| NC_SMTP_PORT | For SMTP plugin - SMTP port value | |
| NC_SMTP_USERNAME | For SMTP plugin (Optional) - SMTP username value for authentication | |
| NC_SMTP_PASSWORD | For SMTP plugin (Optional) - SMTP password value for authentication | |
| NC_SMTP_SECURE | For SMTP plugin (Optional) - To enable secure set value as `true` any other value treated as false | |
| NC_SMTP_IGNORE_TLS | For SMTP plugin (Optional) - To ignore tls set value as `true` any other value treated as false. For more info visit https://nodemailer.com/smtp/ | |
| NC_S3_BUCKET_NAME | For S3 storage plugin - AWS S3 bucket name | |
| NC_S3_REGION | For S3 storage plugin - AWS S3 region | |
| NC_S3_ACCESS_KEY | For S3 storage plugin - AWS access key credential for accessing resource | |
| NC_S3_ACCESS_SECRET | For S3 storage plugin - AWS access secret credential for accessing resource | |
| NC_ATTACHMENT_FIELD_SIZE | For setting the attachment field size(in Bytes) | Defaults to 20MB |
| NC_MAX_ATTACHMENTS_ALLOWED | Maximum Number of attachments per cell | |
| NC_ADMIN_EMAIL | For updating/creating super admin with provided email and password | |
| NC_ADMIN_PASSWORD | For updating/creating super admin with provided email and password. Your password should have at least 8 letters with one uppercase, one number and one special letter(Allowed special chars $&+,:;=?@#\|'.^*()%!_-" ) | |
| NODE_OPTIONS | For passing Node.js [options](https://nodejs.org/api/cli.html#node_optionsoptions) to instance | |
| NC_MINIMAL_DBS | Create a new SQLite file for each project. All the db files are stored in `nc_minimal_dbs` folder in current working directory. (This option restricts project creation on external sources) | |
| NC_DISABLE_AUDIT | Disable Audit Log | `false` |
| NC_AUTOMATION_LOG_LEVEL | Possible Values: `OFF`, `ERROR`, `ALL`. See [Webhooks](/automation/webhook/create-webhook#call-log) for details. | `OFF` |
| NC_SECURE_ATTACHMENTS | Allow accessing attachments only through presigned urls. To enable set value as `true` any other value treated as false. (⚠ this will make existing links inaccessible ⚠) | `false` |
| NC_ATTACHMENT_EXPIRE_SECONDS | How many seconds before expiring presigned attachment urls. (Attachments will expire in at least set seconds and at most 10mins after set time) | 7200 (2 hours) |
| NC_ALLOW_LOCAL_HOOKS | To enable set value as `true` any other value treated as false. (⚠ this will allow webhooks to call local links which can raise security issues ⚠) | `false` |
| NC_SANITIZE_COLUMN_NAME | Sanitize the column name during column creation. To enable set value as `true` any other value treated as false. | `true` |
| Variable | Description | If absent |
| -------- | ----------- | --------- |
| `NC_DB` | See our example database URLs [here](https://github.com/nocodb/nocodb#docker). | A local SQLite database will be created in root folder if `NC_DB` is not provided |
| `NC_DB_JSON` | Can be used instead of `NC_DB` and value should be valid knex connection JSON | |
| `NC_DB_JSON_FILE` | Can be used instead of `NC_DB` and value should be a valid path to knex connection JSON | |
| `DATABASE_URL` | Can be used instead of `NC_DB` and value should be in JDBC URL format | |
| `DATABASE_URL_FILE` | Can be used instead of `DATABASE_URL` and value should be a valid path to file containing JDBC URL format. | |
| `NC_AUTH_JWT_SECRET` | JWT secret used for auth and storing other secrets | A random secret will be generated |
| `PORT` | For setting app running port | `8080` |
| `DB_QUERY_LIMIT_DEFAULT` | Pagination limit | `25` |
| `DB_QUERY_LIMIT_GROUP_BY_GROUP` | Group per page limit | `10` |
| `DB_QUERY_LIMIT_GROUP_BY_RECORD` | Record per group limit | `10` |
| `DB_QUERY_LIMIT_MAX` | Maximum allowed pagination limit | `1000` |
| `DB_QUERY_LIMIT_MIN` | Minimum allowed pagination limit | `1` |
| `NC_TOOL_DIR` | App directory to keep metadata and app related files | Defaults to current working directory. In docker maps to `/usr/app/data/` for mounting volume. |
| `NC_PUBLIC_URL` | Used for sending Email invitations | Best guess from http request params |
| `NC_JWT_EXPIRES_IN` | JWT token expiry time | `10h` |
| `NC_CONNECT_TO_EXTERNAL_DB_DISABLED` | Disable Project creation with external database | |
| `NC_INVITE_ONLY_SIGNUP` | Removed since version 0.99.0 and now it's recommended to use [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup). Allow users to signup only via invite URL, value should be any non-empty string. | |
| `NUXT_PUBLIC_NC_BACKEND_URL` | Custom Backend URL | `http://localhost:8080` will be used |
| `NC_REQUEST_BODY_SIZE` | Request body size [limit](https://expressjs.com/en/resources/middleware/body-parser.html#limit) | `1048576` |
| `NC_EXPORT_MAX_TIMEOUT` | After `NC_EXPORT_MAX_TIMEOUT`, CSV gets downloaded in batches | Default value `5000` (in milliseconds) will be used |
| `NC_DISABLE_TELE` | Disable telemetry | |
| `NC_DASHBOARD_URL` | Custom dashboard URL path | `/dashboard` |
| `NC_GOOGLE_CLIENT_ID` | Google client ID to enable Google authentication | |
| `NC_GOOGLE_CLIENT_SECRET` | Google client secret to enable Google authentication | |
| `NC_MIGRATIONS_DISABLED` | Disable NocoDB migration | |
| `NC_MIN` | If set to any non-empty string the default splash screen (initial welcome animation) and matrix screensaver will disable | |
| `NC_SENTRY_DSN` | For Sentry monitoring | |
| `NC_REDIS_URL` | Custom Redis URL. Example: `redis://:authpassword@127.0.0.1:6380/4` | Meta data will be stored in memory |
| `NC_DISABLE_ERR_REPORT` | Disable error reporting | |
| `NC_DISABLE_CACHE` | To be used only while debugging. On setting this to `true` - meta data be fetched from db instead of redis/cache. | `false` |
| `NC_SMTP_FROM` | For SMTP plugin - Email sender address | |
| `NC_SMTP_HOST` | For SMTP plugin - SMTP host value | |
| `NC_SMTP_PORT` | For SMTP plugin - SMTP port value | |
| `NC_SMTP_USERNAME` | For SMTP plugin (Optional) - SMTP username value for authentication | |
| `NC_SMTP_PASSWORD` | For SMTP plugin (Optional) - SMTP password value for authentication | |
| `NC_SMTP_SECURE` | For SMTP plugin (Optional) - To enable secure set value as `true` any other value treated as false | |
| `NC_SMTP_IGNORE_TLS` | For SMTP plugin (Optional) - To ignore tls set value as `true` any other value treated as false. For more info visit https://nodemailer.com/smtp/ | |
| `NC_S3_BUCKET_NAME` | For S3 storage plugin - AWS S3 bucket name | |
| `NC_S3_REGION` | For S3 storage plugin - AWS S3 region | |
| `NC_S3_ACCESS_KEY` | For S3 storage plugin - AWS access key credential for accessing resource | |
| `NC_S3_ACCESS_SECRET` | For S3 storage plugin - AWS access secret credential for accessing resource | |
| `NC_ATTACHMENT_FIELD_SIZE` | For setting the attachment field size(in Bytes) | Defaults to 20MB |
| `NC_MAX_ATTACHMENTS_ALLOWED` | Maximum Number of attachments per cell | |
| `NC_ADMIN_EMAIL` | For updating/creating super admin with provided email and password | |
| `NC_ADMIN_PASSWORD` | For updating/creating super admin with provided email and password. Your password should have at least 8 letters with one uppercase, one number and one special letter. Allowed special characters include `$&+,:;=?@#\|'.^*()%!_-"`. ) | |
| `NODE_OPTIONS` | For passing Node.js [options](https://nodejs.org/api/cli.html#node_optionsoptions) to instance | |
| `NC_MINIMAL_DBS` | Create a new SQLite file for each project. All the db files are stored in `nc_minimal_dbs` folder in current working directory. (This option restricts project creation on external sources) | |
| `NC_DISABLE_AUDIT` | Disable Audit Log | `false` |
| `NC_AUTOMATION_LOG_LEVEL` | Possible Values: `OFF`, `ERROR`, `ALL`. See [Webhooks](/automation/webhook/create-webhook#call-log) for details. | `OFF` |
| `NC_SECURE_ATTACHMENTS` | Allow accessing attachments only through presigned urls. To enable set value as `true` any other value treated as false. (⚠ this will make existing links inaccessible ⚠) | `false` |
| `NC_ATTACHMENT_EXPIRE_SECONDS` | How many seconds before expiring presigned attachment urls. (Attachments will expire in at least set seconds and at most 10mins after set time) | 7200 (2 hours) |
| `NC_ALLOW_LOCAL_HOOKS` | To enable set value as `true` any other value treated as false. (⚠ this will allow webhooks to call local links which can raise security issues ⚠) | `false` |
| `NC_SANITIZE_COLUMN_NAME` | Sanitize the column name during column creation. To enable set value as `true` any other value treated as false. | `true` |
| `LITESTREAM_S3_ENDPOINT` | URL of an S3-compatible object storage service endpoint like `s3.eu-central-1.amazonaws.com`. | *Litestream replication is disabled if this variable is not set.* |
| `LITESTREAM_S3_BUCKET` | Name of the S3-compatible object storage bucket to store the Litestream replication in. | *Litestream replication is disabled if this variable is not set.* |
| `LITESTREAM_S3_PATH` | Directory path to use within the Litestream replication bucket. | Defaults to `nocodb`. |
| `LITESTREAM_S3_ACCESS_KEY_ID` | Litestream authentication key for the S3 replica. | *Litestream replication is disabled if this variable is not set.* |
| `LITESTREAM_S3_SECRET_ACCESS_KEY` | Litestream authentication key for the S3 replica. | *Litestream replication is disabled if this variable is not set.* |
| `LITESTREAM_S3_SKIP_VERIFY` | Whether to disable TLS verification. This is useful when testing against a local node such as MinIO and you are using self-signed certificates. | Defaults to `false`. |
| `LITESTREAM_RETENTION` | Amount of time Litestream snapshot and WAL files are kept. After the retention period, a new snapshot is created and the old one is removed. WAL files that exist before the oldest snapshot will also be removed. | Defaults to `1440h` (60 days). |
| `LITESTREAM_RETENTION_CHECK_INTERVAL` | Frequency in which Litestream will check if retention needs to be enforced. | Defaults to `72h` (3 days). |
| `LITESTREAM_SNAPSHOT_INTERVAL` | Frequency in which new Litestream snapshots are created. A higher frequency reduces the time to restore since newer snapshots will have fewer WAL frames to apply. Retention still applies to these snapshots. | Defaults to `24h` (1 day). |
| `LITESTREAM_SYNC_INTERVAL` | Frequency in which frames are pushed to the Litestream replica. Increasing this frequency can increase object storage costs significantly. | Defaults to `60s` (1 minute). |

17
packages/nocodb/Dockerfile

@ -52,17 +52,24 @@ RUN pnpm install --prod --shamefully-hoist \
FROM alpine:3.19
WORKDIR /usr/src/app
ENV NC_DOCKER 0.6
ENV NODE_ENV production
ENV PORT 8080
ENV NC_TOOL_DIR=/usr/app/data/
ENV LITESTREAM_S3_SKIP_VERIFY=false \
LITESTREAM_S3_PATH=nocodb \
LITESTREAM_RETENTION=1440h \
LITESTREAM_RETENTION_CHECK_INTERVAL=72h \
LITESTREAM_SNAPSHOT_INTERVAL=24h \
LITESTREAM_SYNC_INTERVAL=60s \
NC_DOCKER=0.6 \
NC_TOOL_DIR=/usr/app/data/ \
NODE_ENV=production \
PORT=8080
RUN apk --update --no-cache add \
nodejs \
dumb-init
# Copy litestream binary
# Copy litestream binary and config file
COPY --from=lt-builder /usr/src/lt /usr/local/bin/litestream
COPY ./docker/litestream.yml /etc/litestream.yml
# Copy production code & main entry file
COPY --from=builder /usr/src/app/ /usr/src/app/
COPY --from=builder /usr/src/appEntry/ /usr/src/appEntry/

22
packages/nocodb/docker/litestream.yml

@ -0,0 +1,22 @@
# Docs: https://litestream.io/reference/config/
dbs:
- path: ${NC_TOOL_DIR}noco.db
replicas:
- type: s3
endpoint: ${LITESTREAM_S3_ENDPOINT}
force-path-style: true
skip-verify: ${LITESTREAM_S3_SKIP_VERIFY}
bucket: ${LITESTREAM_S3_BUCKET}
path: ${LITESTREAM_S3_PATH}
access-key-id: ${LITESTREAM_S3_ACCESS_KEY_ID}
secret-access-key: ${LITESTREAM_S3_SECRET_ACCESS_KEY}
retention: ${LITESTREAM_RETENTION}
retention-check-interval: ${LITESTREAM_RETENTION_CHECK_INTERVAL}
snapshot-interval: ${LITESTREAM_SNAPSHOT_INTERVAL}
sync-interval: ${LITESTREAM_SYNC_INTERVAL}
# age:
# identities:
# - ${LITESTREAM_AGE_SECRET_KEY}
# recipients:
# - ${LITESTREAM_AGE_PUBLIC_KEY}

14
packages/nocodb/docker/start-litestream.sh

@ -6,21 +6,21 @@ if [ -n "${NC_TOOL_DIR}" ]; then
mkdir -p "$NC_TOOL_DIR"
fi
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${AWS_SECRET_ACCESS_KEY}" ] && [ -n "${AWS_BUCKET}" ] && [ -n "${AWS_BUCKET_PATH}" ]; then
if [ -n "${LITESTREAM_S3_ENDPOINT}" ] && [ -n "${LITESTREAM_S3_BUCKET}" ] && [ -n "${LITESTREAM_ACCESS_KEY_ID}" ] && [ -n "${LITESTREAM_SECRET_ACCESS_KEY}" ] ; then
if [ -f "${NC_TOOL_DIR}noco.db" ]
then
if [ -f "${NC_TOOL_DIR}noco.db" ] ; then
rm "${NC_TOOL_DIR}noco.db"
rm "${NC_TOOL_DIR}noco.db-shm"
rm "${NC_TOOL_DIR}noco.db-wal"
fi
litestream restore -o "${NC_TOOL_DIR}noco.db" "s3://$AWS_BUCKET/$AWS_BUCKET_PATH"
if [ ! -f "${NC_TOOL_DIR}noco.db" ]
then
litestream restore "${NC_TOOL_DIR}noco.db"
if [ ! -f "${NC_TOOL_DIR}noco.db" ] ; then
touch "${NC_TOOL_DIR}noco.db"
fi
litestream replicate "${NC_TOOL_DIR}noco.db" "s3://$AWS_BUCKET/$AWS_BUCKET_PATH" &
litestream replicate &
fi
node docker/main.js

22
packages/nocodb/litestream/Dockerfile

@ -38,12 +38,6 @@ RUN chmod +x /usr/src/appEntry/start.sh
FROM alpine:3.19
#ENV AWS_ACCESS_KEY_ID=
#ENV AWS_SECRET_ACCESS_KEY=
#ENV AWS_BUCKET=
#WORKDIR /usr/src/
#
## Install go lang
@ -68,10 +62,15 @@ FROM alpine:3.19
WORKDIR /usr/src/app
ENV NC_DOCKER 0.6
ENV PORT 8080
ENV NC_TOOL_DIR=/usr/app/data/
ENV LITESTREAM_S3_SKIP_VERIFY=false \
LITESTREAM_S3_PATH=nocodb \
LITESTREAM_RETENTION=1440h \
LITESTREAM_RETENTION_CHECK_INTERVAL=72h \
LITESTREAM_SNAPSHOT_INTERVAL=24h \
LITESTREAM_SYNC_INTERVAL=60s \
NC_DOCKER=0.6 \
NC_TOOL_DIR=/usr/app/data/ \
PORT=8080
# Copy application dependency manifests to the container image.
# A wildcard is used to ensure both package.json AND package-lock.json are copied.
@ -84,8 +83,9 @@ RUN apk --update --no-cache add \
nodejs \
tar
# Copy litestream binary
# Copy litestream binary and config file
COPY --from=lt /usr/src/lt /usr/local/bin/litestream
COPY ./docker/litestream.yml /etc/litestream.yml
# Copy production code & main entry file
COPY --from=builder /usr/src/app/ /usr/src/app/
COPY --from=builder /usr/src/appEntry/ /usr/src/appEntry/

Loading…
Cancel
Save