feat: add litestream support

re #540

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb/Dockerfile

@@ -1,3 +1,20 @@
+###########
+# Litestream Builder
+###########
+FROM golang:alpine3.14 as lt-builder
+
+WORKDIR /usr/src/
+
+RUN apk add --no-cache git make musl-dev gcc
+
+# build litestream
+RUN git clone https://github.com/benbjohnson/litestream.git litestream
+RUN cd litestream ; go install ./cmd/litestream
+
+RUN cp $GOPATH/bin/litestream /usr/src/lt
+
+
+
 ###########
 # Builder
 ###########
@@ -9,7 +26,8 @@ WORKDIR /usr/src/app
 # Copying this separately prevents re-running npm ci on every code change.
 COPY ./package*.json ./
 COPY ./docker/main.js ./docker/main.js
-COPY ./docker/start.sh /usr/src/appEntry/start.sh
+#COPY ./docker/start.sh /usr/src/appEntry/start.s
+COPY ./docker/start-litestream.sh /usr/src/appEntry/start.sh
 
 # install production dependencies,
 # reduce node_module size with modclean & removing sqlite deps,
@@ -34,6 +52,8 @@ RUN apk --update --no-cache add \
     nodejs \
     tar
 
+# Copy litestream binary build
+COPY  --from=lt-builder /usr/src/lt /usr/src/appEntry/litestream
 # Copy packaged production code & main entry file
 COPY --from=builder /usr/src/appEntry/ /usr/src/appEntry/
 

packages/nocodb/docker/start-litestream.sh

@@ -3,14 +3,25 @@
 FILE="/usr/src/app/package.json"
 #sleep 5
 
+if [ ! -z "${NC_TOOL_DIR}"  ]; then
+  mkdir -p $NC_TOOL_DIR
+fi
+
+if [[ ! -z "${AWS_ACCESS_KEY_ID}" && ! -z "${AWS_SECRET_ACCESS_KEY}" && ! -z "${AWS_BUCKET}" && ! -z "${AWS_BUCKET_PATH}" ]]; then
+
+  if [ -f "${NC_TOOL_DIR}noco.db" ]
+  then
+    rm "${NC_TOOL_DIR}noco.db"
+    rm "${NC_TOOL_DIR}noco.db-shm"
+    rm "${NC_TOOL_DIR}noco.db-wal"
+  fi
 
-if [[ ! -z $AWS_ACCESS_KEY_ID && ! -z $AWS_SECRET_ACCESS_KEY && ! -z $AWS_BUCKET ]]; then
+  /usr/src/appEntry/litestream restore -o "${NC_TOOL_DIR}noco.db" s3://$AWS_BUCKET/$AWS_BUCKET_PATH;
-  /usr/src/appEntry/litestream restore -o noco.db s3://$AWS_BUCKET/noco.db;
+  if [ ! -f "${NC_TOOL_DIR}noco.db" ]
-  if [ ! -f "noco.db" ]
   then
-    touch noco.db
+    touch "${NC_TOOL_DIR}noco.db"
   fi
-  /usr/src/appEntry/litestream replicate noco.db s3://$AWS_BUCKET/noco.db &
+  /usr/src/appEntry/litestream replicate "${NC_TOOL_DIR}noco.db" s3://$AWS_BUCKET/$AWS_BUCKET_PATH &
 fi
 
 if [ ! -f "$FILE" ]
@@ -18,4 +29,4 @@ then
   tar -xzf /usr/src/appEntry/app.tar.gz -C /usr/src/app/
 fi
 
-DEBUG=xc* node docker/main.js
+node docker/main.js

packages/nocodb/litestream/Dockerfile

@@ -1,4 +1,40 @@
-FROM alpine:3.12
+FROM golang:alpine3.14 as lt
+
+WORKDIR /usr/src/
+
+RUN apk add --no-cache git make musl-dev gcc
+
+# build litestream
+RUN git clone https://github.com/benbjohnson/litestream.git litestream
+RUN cd litestream ; go install ./cmd/litestream
+
+RUN cp $GOPATH/bin/litestream /usr/src/lt
+
+
+
+FROM node:12 as builder
+WORKDIR /usr/src/app
+
+# Copy application dependency manifests to the container image.
+# A wildcard is used to ensure both package.json AND package-lock.json are copied.
+# Copying this separately prevents re-running npm ci on every code change.
+COPY ./package*.json ./
+COPY ./docker/main.js ./docker/main.js
+#COPY ./docker/start.sh /usr/src/appEntry/start.sh
+COPY ./docker/start-litestream.sh /usr/src/appEntry/start.sh
+# install production dependencies,
+# reduce node_module size with modclean & removing sqlite deps,
+# package built code into app.tar.gz & add execute permission to start.sh
+RUN npm ci --production --quiet
+RUN npx modclean --patterns="default:*" --ignore="nc-lib-gui/**,dayjs/**,express-status-monitor/**" --run
+RUN rm -rf ./node_modules/sqlite3/deps
+RUN tar -czf ../appEntry/app.tar.gz ./*
+RUN chmod +x /usr/src/appEntry/start.sh
+
+
+
+
+FROM alpine:3.14
 
 #ENV AWS_ACCESS_KEY_ID=
 #ENV AWS_SECRET_ACCESS_KEY=
@@ -28,41 +64,29 @@ FROM alpine:3.12
 #RUN apk --update --no-cache add paxctl \
 #	&& paxctl -cm $(which node)
 
+WORKDIR /usr/src/app
+
+ENV NC_DOCKER 0.6
 ENV PORT 8080
-ENV NODE_ENV=dev
+ENV NC_TOOL_DIR=/usr/app/data/
-#ENV TOOL_DIR=/tool
-ENV NC_VERSION=0.6
 
-# Create and change to the app directory.
-WORKDIR /usr/src/appTemp
 
 # Copy application dependency manifests to the container image.
 # A wildcard is used to ensure both package.json AND package-lock.json are copied.
 # Copying this separately prevents re-running npm install on every code change.
 #COPY ./build/ ./build/
-COPY ./docker/main.js ./docker/main.js
+#COPY ./docker/main.js ./docker/main.js
-COPY ./package.json ./
+#COPY ./package.json ./
-COPY ./docker/start-litestream.sh /usr/src/appEntry/start.sh
-COPY ./litestream/litestream /usr/src/appEntry/litestream
 
-# Install production dependencies.
-#RUN npm i ../xc-lib-gui/ && npm install --cache=/usr/src/app/cache --production && npx modclean  --patterns="default:*" --run && rm -rf /usr/src/app/cache && rm -rf /root/.npm
 RUN apk --update --no-cache add \
     nodejs \
-    	nodejs-npm \
+    tar
-    	tar\
+
-   && npm install --cache=/usr/src/app/cache --production \
+# Copy litestream binary build
-    && npx modclean --patterns="default:*" --ignore="xc-lib-gui/**,dayjs/**,express-status-monitor/**" --run  \
+COPY  --from=lt /usr/src/lt /usr/src/appEntry/litestream
-    && rm -rf ./node_modules/sqlite3/deps \
+# Copy packaged production code & main entry file
-    && rm -rf ./node_modules/xc-lib-gui/lib/dist/_nuxt \
+COPY --from=builder /usr/src/appEntry/ /usr/src/appEntry/
-    && rm -rf /usr/src/app/cache && rm -rf /root/.npm \
+
-   && apk del nodejs-npm \
-   && tar -czf ../appEntry/app.tar.gz ./* ; rm -rf ./* && chmod +x /usr/src/appEntry/start.sh
-
-
-#COPY ./node_modules/xc-lib/ ./node_modules/xc-lib/
-#COPY ./node_modules/xc-lib-gui/ ./node_modules/xc-lib-gui/
-WORKDIR /usr/src/app
 
 # Run the web service on container startup.
 #CMD [ "node", "docker/index.js" ]