From 1c273c2a75c538b1ea8d3fa99692cbceb87830cb Mon Sep 17 00:00:00 2001 From: starbirdtech383 Date: Mon, 29 Jan 2024 22:53:38 +0530 Subject: [PATCH] refactor to aio and add all env variables prompt --- docker-compose/{full-install => aio}/.env | 0 .../{full-install => aio}/.gitignore | 0 .../{full-install => aio}/README.md | 0 docker-compose/aio/advanced.md | 46 ++++++++++++ docker-compose/aio/bin/nginx_reload.sh | 2 + .../{full-install => aio}/bin/nginx_start.sh | 0 docker-compose/aio/conf/nc_properties.env | 16 +++++ .../{full-install => aio}/docker-compose.yml | 5 ++ .../{full-install => aio}/install.sh | 0 .../conf-templates/certbot_conf.template | 0 .../ssl_server_name_conf.template | 0 .../nginx/conf.d/backends.conf | 0 .../nginx/conf.d/default.conf | 0 .../nginx/conf.d/fe_artf.conf | 0 .../nginx/conf/nginx.conf | 0 .../{full-install => aio}/pre-req-check.sh | 0 docker-compose/aio/prepare_env.sh | 70 +++++++++++++++++++ .../sbin/gen_letsencrypt_cert.sh | 0 docker-compose/aio/sbin/install_vars.sh | 15 ++++ .../{full-install => aio}/sbin/renew_certs.sh | 0 .../{full-install => aio}/sbin/util.sh | 0 .../{full-install => aio}/security.md | 0 docker-compose/full-install/advanced.md | 23 ------ .../full-install/bin/nginx_reload.sh | 1 - docker-compose/full-install/prepare_env.sh | 41 ----------- .../full-install/sbin/install_vars.sh | 7 -- 26 files changed, 154 insertions(+), 72 deletions(-) rename docker-compose/{full-install => aio}/.env (100%) rename docker-compose/{full-install => aio}/.gitignore (100%) rename docker-compose/{full-install => aio}/README.md (100%) create mode 100644 docker-compose/aio/advanced.md create mode 100755 docker-compose/aio/bin/nginx_reload.sh rename docker-compose/{full-install => aio}/bin/nginx_start.sh (100%) create mode 100644 docker-compose/aio/conf/nc_properties.env rename docker-compose/{full-install => aio}/docker-compose.yml (94%) rename docker-compose/{full-install => aio}/install.sh (100%) rename docker-compose/{full-install => aio}/nginx/conf-templates/certbot_conf.template (100%) rename docker-compose/{full-install => aio}/nginx/conf-templates/ssl_server_name_conf.template (100%) rename docker-compose/{full-install => aio}/nginx/conf.d/backends.conf (100%) rename docker-compose/{full-install => aio}/nginx/conf.d/default.conf (100%) rename docker-compose/{full-install => aio}/nginx/conf.d/fe_artf.conf (100%) rename docker-compose/{full-install => aio}/nginx/conf/nginx.conf (100%) rename docker-compose/{full-install => aio}/pre-req-check.sh (100%) create mode 100755 docker-compose/aio/prepare_env.sh rename docker-compose/{full-install => aio}/sbin/gen_letsencrypt_cert.sh (100%) create mode 100644 docker-compose/aio/sbin/install_vars.sh rename docker-compose/{full-install => aio}/sbin/renew_certs.sh (100%) rename docker-compose/{full-install => aio}/sbin/util.sh (100%) rename docker-compose/{full-install => aio}/security.md (100%) delete mode 100644 docker-compose/full-install/advanced.md delete mode 100755 docker-compose/full-install/bin/nginx_reload.sh delete mode 100755 docker-compose/full-install/prepare_env.sh delete mode 100644 docker-compose/full-install/sbin/install_vars.sh diff --git a/docker-compose/full-install/.env b/docker-compose/aio/.env similarity index 100% rename from docker-compose/full-install/.env rename to docker-compose/aio/.env diff --git a/docker-compose/full-install/.gitignore b/docker-compose/aio/.gitignore similarity index 100% rename from docker-compose/full-install/.gitignore rename to docker-compose/aio/.gitignore diff --git a/docker-compose/full-install/README.md b/docker-compose/aio/README.md similarity index 100% rename from docker-compose/full-install/README.md rename to docker-compose/aio/README.md diff --git a/docker-compose/aio/advanced.md b/docker-compose/aio/advanced.md new file mode 100644 index 0000000000..3c6bbb93ed --- /dev/null +++ b/docker-compose/aio/advanced.md @@ -0,0 +1,46 @@ +# Advanced operations + +## Restarting containers +There are atleast 4 main containers which are running as part of this installation through same docker-compose. The same docker-compose can be leveraged to restart any or all of these containers. + +Use below command to restart all containers\ +``` docker compose restart ``` + +To restart individual containers with name ( names: nocodb, nginx, postgres, redis)\ +ex: to restart nginx\ +``` docker compose restart nginx ``` + +## reload nginx +use utility script at [./bin/nginx_reload.sh](./bin/nginx_reload.sh) + +## Upgrade nocodb instance + +## Enable SSL +To enable SSL for incoming https requests, nginx should be configured with combination of a public certificate and a private key. The SSL private key is kept secret on the server. It will be used to encrypt content sent to clients. +Below are different approaches to get and configure certificates. Make your choice +### letsencrypt for generating certificates +Certificates/key can be obtained by trusted CA (Certificate Authorities), there are many paid vendors found online or you can also use [letsencrypt](https://letsencrypt.org/) a non profit certificate provider for free however we recommend [https://www.abetterinternet.org/donate/](donate) for their service. + +### Bring your own certificates +If you already have the certificates, either self signed or generated by any other means, you will need to configure them with nginx. Below are the steps +TBD +### self signed certificates +One of the pre-requisite is that your server should be associated with the domain name. In the absence of that you could use self signed certificates which does ecrypt but browsers show warning. + +## Database password rotation +As a security measure, It is best practice to rotate the database credentials periodically. Assuming you would have created new credentials in postgres database. The db credentials are persisted on filesystem as part of initial install and it will be available at +[./conf/nc_properties.env](./conf/nc_properties.env)\ +update properties POSTGRES_USER, POSTGRES_PASSWORD with new credentials and [restarting nocodb](#restarting-containers) with\ +```docker compose restart nocodb``` + +## nginx configurations +There are two main directories where nginx configurations are maintained +- nocodb team managed configurations at [nginx/conf.d](./conf/nginx/conf.d). +- self managed (you) [conf/nginx/conf.d](./conf/nginx/conf.d) + + +## Add Node + +## Minio integration for resources + +## Enable scheduled backups to run nightly on all your instances. \ No newline at end of file diff --git a/docker-compose/aio/bin/nginx_reload.sh b/docker-compose/aio/bin/nginx_reload.sh new file mode 100755 index 0000000000..3e591ee95b --- /dev/null +++ b/docker-compose/aio/bin/nginx_reload.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker exec -it nginx /etc/init.d/nginx reload \ No newline at end of file diff --git a/docker-compose/full-install/bin/nginx_start.sh b/docker-compose/aio/bin/nginx_start.sh similarity index 100% rename from docker-compose/full-install/bin/nginx_start.sh rename to docker-compose/aio/bin/nginx_start.sh diff --git a/docker-compose/aio/conf/nc_properties.env b/docker-compose/aio/conf/nc_properties.env new file mode 100644 index 0000000000..848cb06df3 --- /dev/null +++ b/docker-compose/aio/conf/nc_properties.env @@ -0,0 +1,16 @@ +# Environment Variables +POSTGRES_USER=postgres +POSTGRES_PASSWORD=test123 +POSTGRES_DB=nocodb +NC_REDIS_URL=redis://redis:6379/4 +NC_DB=pg://postgres:5432?u=postgres&password=${POSTGRES_PASSWORD:-nocodb}&d=postgres +NC_PUBLIC_URL=rajanishs-MacBook-Pro.local +NC_CONNECT_TO_EXTERNAL_DB_DISABLED=false +NC_INVITE_ONLY_SIGNUP=false +NC_ADMIN_EMAIL=false +NC_ADMIN_PASSWORD=false +NC_S3_BUCKET_NAME=asdf +NC_S3_REGION=asd +NC_S3_ACCESS_KEY=sda +NC_S3_ACCESS_SECRET=adsfa + diff --git a/docker-compose/full-install/docker-compose.yml b/docker-compose/aio/docker-compose.yml similarity index 94% rename from docker-compose/full-install/docker-compose.yml rename to docker-compose/aio/docker-compose.yml index 5ba5593488..f0b69d7e01 100644 --- a/docker-compose/full-install/docker-compose.yml +++ b/docker-compose/aio/docker-compose.yml @@ -35,6 +35,11 @@ services: - ${NC_INSTALL_ROOT:-/opt/nocodb}/data/postgres:/var/lib/postgresql/data networks: - nocodb-001 + healthcheck: + interval: 10s + retries: 10 + test: "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}" + timeout: 2s deploy: resources: limits: diff --git a/docker-compose/full-install/install.sh b/docker-compose/aio/install.sh similarity index 100% rename from docker-compose/full-install/install.sh rename to docker-compose/aio/install.sh diff --git a/docker-compose/full-install/nginx/conf-templates/certbot_conf.template b/docker-compose/aio/nginx/conf-templates/certbot_conf.template similarity index 100% rename from docker-compose/full-install/nginx/conf-templates/certbot_conf.template rename to docker-compose/aio/nginx/conf-templates/certbot_conf.template diff --git a/docker-compose/full-install/nginx/conf-templates/ssl_server_name_conf.template b/docker-compose/aio/nginx/conf-templates/ssl_server_name_conf.template similarity index 100% rename from docker-compose/full-install/nginx/conf-templates/ssl_server_name_conf.template rename to docker-compose/aio/nginx/conf-templates/ssl_server_name_conf.template diff --git a/docker-compose/full-install/nginx/conf.d/backends.conf b/docker-compose/aio/nginx/conf.d/backends.conf similarity index 100% rename from docker-compose/full-install/nginx/conf.d/backends.conf rename to docker-compose/aio/nginx/conf.d/backends.conf diff --git a/docker-compose/full-install/nginx/conf.d/default.conf b/docker-compose/aio/nginx/conf.d/default.conf similarity index 100% rename from docker-compose/full-install/nginx/conf.d/default.conf rename to docker-compose/aio/nginx/conf.d/default.conf diff --git a/docker-compose/full-install/nginx/conf.d/fe_artf.conf b/docker-compose/aio/nginx/conf.d/fe_artf.conf similarity index 100% rename from docker-compose/full-install/nginx/conf.d/fe_artf.conf rename to docker-compose/aio/nginx/conf.d/fe_artf.conf diff --git a/docker-compose/full-install/nginx/conf/nginx.conf b/docker-compose/aio/nginx/conf/nginx.conf similarity index 100% rename from docker-compose/full-install/nginx/conf/nginx.conf rename to docker-compose/aio/nginx/conf/nginx.conf diff --git a/docker-compose/full-install/pre-req-check.sh b/docker-compose/aio/pre-req-check.sh similarity index 100% rename from docker-compose/full-install/pre-req-check.sh rename to docker-compose/aio/pre-req-check.sh diff --git a/docker-compose/aio/prepare_env.sh b/docker-compose/aio/prepare_env.sh new file mode 100755 index 0000000000..d574267298 --- /dev/null +++ b/docker-compose/aio/prepare_env.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# prepares env file with all the required env variables. +# + +# -- main line code starts here -- +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source ${SCRIPT_DIR}/sbin/util.sh +source ${SCRIPT_DIR}/sbin/install_vars.sh + +ENV_FILE=${SCRIPT_DIR}/conf/nc_properties.env +bkp_file=${ENV_FILE}-$(date +%s).bak +# Source existing nc_envs.env file to get current values +if [ -f ${ENV_FILE} ]; then + source ${ENV_FILE} + echo "backing up previous ${ENV_FILE} file to ${bkp_file}" + cp ${ENV_FILE} ${bkp_file} +fi + +function acceptProperty(){ + local varDetail="$1" + prompt=$(echo "$varDetail" | cut -d '|' -f1) + prop=$(echo "$varDetail" | cut -d '|' -f2) + key=$(echo "$prop" | cut -d'=' -f1) + default_value="${prop#*=}" + prev_value_or_default=${!key:-${default_value}} + + # echo prompt: ${prompt} + # echo prop: ${prop} + # echo key: ${key} + # echo default_value: ${default_value} + + read -p "Enter value for $key (default: ${prev_value_or_default}): " user_input + + # Use user input or default value if empty + value=${user_input:-$prev_value_or_default} + + # Store key-value pair in a variable + userValues="${userValues}${key}=${value}\n" +} +# Iterate over the properties array and prompt user for input +echo basic_properties : "${basic_properties[@]}" +for multi_property_array in basic_properties invite_only_signup_priorities google_login_properties email_properties s3_attachment_properties ; do + array_name="$multi_property_array[@]" # Name of the array to process + array=("${!array_name}") + # array=("${!multi_property_array}") + echo array : "${array[@]}" + for varDetail in "${array[@]}"; do + echo varDetail : ${varDetail} + prompt=$(echo "$varDetail" | cut -d '|' -f1) + prop=$(echo "$varDetail" | cut -d '|' -f2) + if [[ ${prompt} == "main" ]] + then + echo $prop + if asksure; then + continue + else + break + fi + fi + acceptProperty "${varDetail}" + done +done + +# Write key-value pairs to nc_envs.env file +echo -e "# Environment Variables\n$userValues" > ${ENV_FILE} + +echo "Environment variables written to ${ENV_FILE} file." + +# echo "creating data conf, data and log directories" +# mkdir -p ${INSTALL_ROOT}/conf ${INSTALL_ROOT}/data ${INSTALL_ROOT}/logs diff --git a/docker-compose/full-install/sbin/gen_letsencrypt_cert.sh b/docker-compose/aio/sbin/gen_letsencrypt_cert.sh similarity index 100% rename from docker-compose/full-install/sbin/gen_letsencrypt_cert.sh rename to docker-compose/aio/sbin/gen_letsencrypt_cert.sh diff --git a/docker-compose/aio/sbin/install_vars.sh b/docker-compose/aio/sbin/install_vars.sh new file mode 100644 index 0000000000..dc5aac5a0b --- /dev/null +++ b/docker-compose/aio/sbin/install_vars.sh @@ -0,0 +1,15 @@ +nocodb_install_version="1.0.0" # Replace with actual version +REQUIRED_PORTS=(80 443) +DOCKER_IMAGES=("redis:latest" "postgres:14.7" "nocodb/nocodb:latest" "nginx" "certbot/certbot:latest" ) + +# Array of properties with default values +basic_properties=("main|basic configurations, customise ?" "Username for postgres database|POSTGRES_USER=postgres" "|POSTGRES_PASSWORD=test123" "|POSTGRES_DB=nocodb" "|NC_REDIS_URL=redis://redis:6379/4" '|NC_DB=pg://postgres:5432?u=postgres&password=${POSTGRES_PASSWORD:-nocodb}&d=postgres' "Are you using custom DNS, configure NC_PUBLIC_URL to reflect in the invite emails?|NC_PUBLIC_URL=$(hostname)" "Disable connecting to external db?|NC_CONNECT_TO_EXTERNAL_DB_DISABLED=false") +invite_only_signup_priorities=("main|Allow invite only sign-up" "invite only signup?|NC_INVITE_ONLY_SIGNUP=false" "|NC_ADMIN_EMAIL=false" "|NC_ADMIN_PASSWORD=false") +google_login_properties=("main|Configure google login" "Enter Client ID|NC_GOOGLE_CLIENT_ID=" "Enter Client ID|NC_GOOGLE_CLIENT_SECRET=") +email_properties=("main|Configure smtp properties" "|NC_SMTP_FROM=" "|NC_SMTP_HOST=" "|NC_SMTP_PORT=" "|NC_SMTP_USERNAME=" "|NC_SMTP_PASSWORD=" "|NC_SMTP_SECURE=" "|NC_SMTP_IGNORE_TLS=" ) +s3_attachment_properties=("main|Do you want to configure s3 for attachements?" "|NC_S3_BUCKET_NAME=nocodb-attachements" "|NC_S3_REGION=" "|NC_S3_ACCESS_KEY=" "|NC_S3_ACCESS_SECRET=" ) + +multi_property_array=(basic_properties invite_only_signup_priorities google_login_properties email_properties s3_attachment_properties) + + + diff --git a/docker-compose/full-install/sbin/renew_certs.sh b/docker-compose/aio/sbin/renew_certs.sh similarity index 100% rename from docker-compose/full-install/sbin/renew_certs.sh rename to docker-compose/aio/sbin/renew_certs.sh diff --git a/docker-compose/full-install/sbin/util.sh b/docker-compose/aio/sbin/util.sh similarity index 100% rename from docker-compose/full-install/sbin/util.sh rename to docker-compose/aio/sbin/util.sh diff --git a/docker-compose/full-install/security.md b/docker-compose/aio/security.md similarity index 100% rename from docker-compose/full-install/security.md rename to docker-compose/aio/security.md diff --git a/docker-compose/full-install/advanced.md b/docker-compose/full-install/advanced.md deleted file mode 100644 index f6cd5120dd..0000000000 --- a/docker-compose/full-install/advanced.md +++ /dev/null @@ -1,23 +0,0 @@ -# Advanced operations - - -## Restarting containers - -## Upgrade nocodb instance - -## Enable SSL -In order to enable SSL for incoming https requests, you will need to have certs which is -private-key public-key pair -### letsencrypt for generating certificates -### Bring your own certificates -### self signed certificates - -## Database pasword rotation - -## nginx configurations - -## Add Node - -## Minio integration for resources - -## Enable scheduled backups to run nightly on all your instances. \ No newline at end of file diff --git a/docker-compose/full-install/bin/nginx_reload.sh b/docker-compose/full-install/bin/nginx_reload.sh deleted file mode 100755 index 27d4d71c41..0000000000 --- a/docker-compose/full-install/bin/nginx_reload.sh +++ /dev/null @@ -1 +0,0 @@ -docker exec -it nginx /etc/init.d/nginx reload \ No newline at end of file diff --git a/docker-compose/full-install/prepare_env.sh b/docker-compose/full-install/prepare_env.sh deleted file mode 100755 index d725642b0c..0000000000 --- a/docker-compose/full-install/prepare_env.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# prepares env file with all the required env variables. -# - -# -- main line code starts here -- -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source ${SCRIPT_DIR}/sbin/util.sh -source ${SCRIPT_DIR}/sbin/install_vars.sh - -ENV_FILE=${SCRIPT_DIR}/conf/nc_properties.env -bkp_file=${ENV_FILE}-$(date +%s).bak -# Source existing nc_envs.env file to get current values -if [ -f ${ENV_FILE} ]; then - source ${ENV_FILE} - echo "backing up previous ${ENV_FILE} file to ${bkp_file}" - cp ${ENV_FILE} ${bkp_file} -fi - -echo "Update or confirm the values to be set" -# Iterate over the properties array and prompt user for input -for prop in "${properties[@]}"; do - key=$(echo "$prop" | cut -d'=' -f1) - default_value="${prop#*=}" - prev_value_or_default=${!key:-${default_value}} - - read -p "Enter value for $key (default: ${prev_value_or_default}): " user_input - - # Use user input or default value if empty - value=${user_input:-$prev_value_or_default} - - # Store key-value pair in a variable - userValues="${userValues}${key}=${value}\n" -done - -# Write key-value pairs to nc_envs.env file -echo -e "# Environment Variables\n$userValues" > ${ENV_FILE} - -echo "Environment variables written to ${ENV_FILE} file." - -echo "creating data conf, data and log directories" -mkdir -p ${INSTALL_ROOT}/conf ${INSTALL_ROOT}/data ${INSTALL_ROOT}/logs diff --git a/docker-compose/full-install/sbin/install_vars.sh b/docker-compose/full-install/sbin/install_vars.sh deleted file mode 100644 index 17135fefb7..0000000000 --- a/docker-compose/full-install/sbin/install_vars.sh +++ /dev/null @@ -1,7 +0,0 @@ -nocodb_install_version="1.0.0" # Replace with actual version -REQUIRED_PORTS=(80 443) -DOCKER_IMAGES=("redis:latest" "postgres:14.7" "nocodb/nocodb:latest" "nginx" "certbot/certbot:latest" ) - -# Array of properties with default values -properties=( "POSTGRES_USER=postgres" "POSTGRES_PASSWORD=test123" "POSTGRES_DB=nocodb" "NC_REDIS_URL=redis://redis:6379/4" 'NC_DB=pg://postgres:5432?u=postgres&password=${POSTGRES_PASSWORD:-nocodb}&d=postgres' ) -# "NC_INSTALL_ROOT=${SCRIPT_DIR}" "MINIO_ROOT_USER=minioadmin" "MINIO_ROOT_PASSWORD=minioadmin" \ No newline at end of file