From 40664e1e4abc59f2159b102b08ca93d2b8d27595 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 12 Oct 2024 01:54:11 +0530 Subject: [PATCH 1/4] chore: optimize docker build time Signed-off-by: Pranav C --- .github/workflows/release-timely-docker.yml | 4 +- packages/nocodb/Dockerfile.timely | 56 +++++++++++++-------- packages/nocodb/package.json | 2 +- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release-timely-docker.yml b/.github/workflows/release-timely-docker.yml index 1841d8562f..7ca567f31a 100644 --- a/.github/workflows/release-timely-docker.yml +++ b/.github/workflows/release-timely-docker.yml @@ -78,7 +78,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 1 ref: ${{ github.ref }} - name: Use Node.js ${{ matrix.node-version }} @@ -144,7 +144,7 @@ jobs: context: ${{ env.working-directory }} file: ${{ env.working-directory }}/Dockerfile.timely build-args: NC_VERSION=${{ steps.get-docker-repository.outputs.DOCKER_BUILD_TAG }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: linux/amd64,linux/arm64 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new push: true diff --git a/packages/nocodb/Dockerfile.timely b/packages/nocodb/Dockerfile.timely index 0cdb946ccf..441c7bc322 100644 --- a/packages/nocodb/Dockerfile.timely +++ b/packages/nocodb/Dockerfile.timely @@ -1,29 +1,31 @@ -# syntax=docker/dockerfile:1 +# syntax=docker/dockerfile:1.5 +# Use Buildx cross-compilation ########### # Litestream Builder ########### -FROM golang:alpine3.19 as lt-builder +FROM --platform=$BUILDPLATFORM golang:alpine3.19 as lt-builder WORKDIR /usr/src/ -RUN apk add --no-cache git make musl-dev gcc +# Use build platform-specific tools +RUN apk --no-cache add git make musl-dev gcc -# build litestream -RUN git clone https://github.com/benbjohnson/litestream.git litestream -RUN cd litestream && go install ./cmd/litestream -RUN cp $GOPATH/bin/litestream /usr/src/lt +# Build litestream for target platform +RUN git clone https://github.com/benbjohnson/litestream.git litestream \ + && cd litestream && GOARCH=$(go env GOARCH) GOOS=$(go env GOOS) go install ./cmd/litestream \ + && cp $GOPATH/bin/litestream /usr/src/lt ########### # Builder ########### -FROM node:18.19.1-alpine as builder +FROM --platform=$BUILDPLATFORM node:18.19.1-alpine as builder WORKDIR /usr/src/app -# install node-gyp dependencies +# Install node-gyp dependencies RUN apk add --no-cache python3 make g++ -# install pnpm +# Install pnpm RUN corepack enable && corepack prepare pnpm@latest --activate # Copy application dependency manifests to the container image. @@ -33,22 +35,33 @@ COPY --link ./docker/start-litestream.sh /usr/src/appEntry/start.sh COPY --link src/public/ ./docker/public/ COPY --link ./docker/nc-gui/ ./docker/nc-gui/ -# for pnpm to generate a flat node_modules without symlinks -# so that modclean could work as expected +# For pnpm to generate a flat node_modules without symlinks +# So that modclean works as expected RUN echo "node-linker=hoisted" > .npmrc -# install production dependencies, -# reduce node_module size with modclean & removing sqlite deps, -# and add execute permission to start.sh +# Install production dependencies, reduce node_module size with modclean +# Removing sqlite deps and add execute permission to start.sh RUN pnpm install --prod --shamefully-hoist \ && pnpm dlx modclean --patterns="default:*" --ignore="nc-lib-gui/**,dayjs/**,express-status-monitor/**,@azure/msal-node/dist/**" --run \ && rm -rf ./node_modules/sqlite3/deps \ && chmod +x /usr/src/appEntry/start.sh +# && (pnpm exec prebuild-install -- --platform=$TARGETPLATFORM --arch=$TARGETARCH -r napi || echo "Prebuilt binary not available") \ +# && npm install sqlite3 --target_arch=$BUILDARCH --target_platform=$BUILDOS --legacy-peer-deps \ -########## + +############ +## Sqlite3 Builder +############ +FROM node:18.19.1-alpine as sqlite-builder +WORKDIR /usr/src/app + +# Install sqlite3 for the target platform to copy to the final image +RUN npm init -y && npm install sqlite3 + +########### # Runner -########## -FROM alpine:3.19 +########### +FROM --platform=$TARGETPLATFORM alpine:3.19 WORKDIR /usr/src/app ENV LITESTREAM_S3_SKIP_VERIFY=false \ @@ -61,16 +74,15 @@ ENV LITESTREAM_S3_SKIP_VERIFY=false \ NODE_ENV=production \ PORT=8080 -RUN apk add --update --no-cache \ - dasel \ - dumb-init \ - nodejs +RUN apk add --update --no-cache dasel dumb-init nodejs # Copy litestream binary and config file COPY --link --from=lt-builder /usr/src/lt /usr/local/bin/litestream COPY --link ./docker/litestream.yml /etc/litestream.yml + # Copy production code & main entry file COPY --link --from=builder /usr/src/app/ /usr/src/app/ +COPY --link --from=sqlite-builder /usr/src/app/node_modules/sqlite3/ /usr/src/app/node_modules/sqlite3/ COPY --link --from=builder /usr/src/appEntry/ /usr/src/appEntry/ EXPOSE 8080 diff --git a/packages/nocodb/package.json b/packages/nocodb/package.json index e9a464a287..a60ddf4497 100644 --- a/packages/nocodb/package.json +++ b/packages/nocodb/package.json @@ -197,4 +197,4 @@ "typescript": "^5.3.3", "webpack-cli": "^5.1.4" } -} \ No newline at end of file +} From ff149552182c7aa1f590d8cc9f293fab151baef6 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 12 Oct 2024 13:43:21 +0530 Subject: [PATCH 2/4] chore: cleanup and add sharp Signed-off-by: Pranav C --- packages/nocodb/Dockerfile.timely | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/nocodb/Dockerfile.timely b/packages/nocodb/Dockerfile.timely index 441c7bc322..b5335cac3c 100644 --- a/packages/nocodb/Dockerfile.timely +++ b/packages/nocodb/Dockerfile.timely @@ -11,9 +11,10 @@ WORKDIR /usr/src/ # Use build platform-specific tools RUN apk --no-cache add git make musl-dev gcc -# Build litestream for target platform +# Build litestream for the target platform RUN git clone https://github.com/benbjohnson/litestream.git litestream \ - && cd litestream && GOARCH=$(go env GOARCH) GOOS=$(go env GOOS) go install ./cmd/litestream \ + && cd litestream \ + && GOARCH=$(echo $TARGETPLATFORM | cut -d '/' -f 2) GOOS=$(echo $TARGETPLATFORM | cut -d '/' -f 1) go install ./cmd/litestream \ && cp $GOPATH/bin/litestream /usr/src/lt ########### @@ -45,18 +46,15 @@ RUN pnpm install --prod --shamefully-hoist \ && pnpm dlx modclean --patterns="default:*" --ignore="nc-lib-gui/**,dayjs/**,express-status-monitor/**,@azure/msal-node/dist/**" --run \ && rm -rf ./node_modules/sqlite3/deps \ && chmod +x /usr/src/appEntry/start.sh -# && (pnpm exec prebuild-install -- --platform=$TARGETPLATFORM --arch=$TARGETARCH -r napi || echo "Prebuilt binary not available") \ -# && npm install sqlite3 --target_arch=$BUILDARCH --target_platform=$BUILDOS --legacy-peer-deps \ - ############ -## Sqlite3 Builder +## Binary Dependencies Builder ############ -FROM node:18.19.1-alpine as sqlite-builder +FROM --platform=$TARGETPLATFORM node:18.19.1-alpine as bin-builder WORKDIR /usr/src/app # Install sqlite3 for the target platform to copy to the final image -RUN npm init -y && npm install sqlite3 +RUN npm init -y && npm install sqlite3 sharp ########### # Runner @@ -82,7 +80,8 @@ COPY --link ./docker/litestream.yml /etc/litestream.yml # Copy production code & main entry file COPY --link --from=builder /usr/src/app/ /usr/src/app/ -COPY --link --from=sqlite-builder /usr/src/app/node_modules/sqlite3/ /usr/src/app/node_modules/sqlite3/ +COPY --link --from=bin-builder /usr/src/app/node_modules/sqlite3/ /usr/src/app/node_modules/sqlite3/ +COPY --link --from=bin-builder /usr/src/app/node_modules/sharp/ /usr/src/app/node_modules/sharp/ COPY --link --from=builder /usr/src/appEntry/ /usr/src/appEntry/ EXPOSE 8080 From 9ab484d610292c9cf063b42f227fb329445db9c0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 12 Oct 2024 14:07:06 +0530 Subject: [PATCH 3/4] fix: tag name correction Signed-off-by: Pranav C --- .github/workflows/release-timely-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-timely-docker.yml b/.github/workflows/release-timely-docker.yml index 7ca567f31a..db7ec228ef 100644 --- a/.github/workflows/release-timely-docker.yml +++ b/.github/workflows/release-timely-docker.yml @@ -63,9 +63,9 @@ jobs: DOCKER_BUILD_TAG=${{ github.event.inputs.currentVersion || inputs.currentVersion }}-${{ github.event.inputs.tag || inputs.tag }} fi if [[ ${{ inputs.isDaily || 'N' }} == 'Y' ]]; then - DOCKER_REPOSITORY=${DOCKER_REPOSITORY}-daily + DOCKER_REPOSITORY=nocodb-daily else - DOCKER_REPOSITORY=${DOCKER_REPOSITORY}-timely + DOCKER_REPOSITORY=nocodb-timely fi fi echo "DOCKER_REPOSITORY=${DOCKER_REPOSITORY}" >> $GITHUB_OUTPUT From 46aaf79ff107c78e378bda574eb2576aa0c8f72b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 14 Oct 2024 11:43:22 +0530 Subject: [PATCH 4/4] chore: extract package versions from package.json and install Signed-off-by: Pranav C --- packages/nocodb/Dockerfile.timely | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/Dockerfile.timely b/packages/nocodb/Dockerfile.timely index b5335cac3c..5db20367fd 100644 --- a/packages/nocodb/Dockerfile.timely +++ b/packages/nocodb/Dockerfile.timely @@ -53,8 +53,15 @@ RUN pnpm install --prod --shamefully-hoist \ FROM --platform=$TARGETPLATFORM node:18.19.1-alpine as bin-builder WORKDIR /usr/src/app +RUN apk add --no-cache jq + +# copy package.json to extract dependency versions +COPY --link ./package.json ./package-copy.json + # Install sqlite3 for the target platform to copy to the final image -RUN npm init -y && npm install sqlite3 sharp +RUN SQLITE3_VERSION=$(jq -r '.dependencies["sqlite3"]' /usr/src/app/package-copy.json) \ + && SHARP_VERSION=$(jq -r '.dependencies["sharp"]' /usr/src/app/package-copy.json) \ + && npm init -y && npm install sqlite3@$SQLITE3_VERSION sharp@$SHARP_VERSION ########### # Runner