Browse Source

Merge pull request #9645 from nocodb/chore/timely-build-without-npm-package

Chore/timely docker build
pull/9654/head
Pranav C 2 months ago committed by GitHub
parent
commit
8b562f9645
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 27
      .github/workflows/release-nightly-dev.yml
  2. 15
      .github/workflows/release-pr.yml
  3. 161
      .github/workflows/release-timely-docker.yml
  4. 80
      packages/nocodb/Dockerfile.timely
  5. 21
      packages/nocodb/src/run/timely.ts
  6. 7
      packages/nocodb/src/services/utils.service.ts
  7. 2
      packages/nocodb/src/utils/TeleBatchProcessor.ts
  8. 1
      packages/nocodb/src/utils/tele.ts
  9. 57
      packages/nocodb/webpack.timely.config.js

27
.github/workflows/release-nightly-dev.yml

@ -43,29 +43,20 @@ jobs:
nightly_build_tag: ${{ steps.tag-step.outputs.NIGHTLY_BUILD_TAG }} nightly_build_tag: ${{ steps.tag-step.outputs.NIGHTLY_BUILD_TAG }}
is_daily: ${{ steps.tag-step.outputs.IS_DAILY }} is_daily: ${{ steps.tag-step.outputs.IS_DAILY }}
current_version: ${{ steps.tag-step.outputs.CURRENT_VERSION }} current_version: ${{ steps.tag-step.outputs.CURRENT_VERSION }}
# Build frontend and backend and publish to npm
release-npm:
needs: set-tag
uses: ./.github/workflows/release-npm.yml
with:
tag: ${{ needs.set-tag.outputs.nightly_build_tag }}
targetEnv: 'DEV'
secrets:
NPM_TOKEN: "${{ secrets.NPM_TOKEN }}"
# Build executables and publish to GitHub # Build executables and publish to GitHub
release-executables: # release-executables:
needs: [set-tag, release-npm] # needs: [set-tag, release-npm]
uses: ./.github/workflows/release-timely-executables.yml # uses: ./.github/workflows/release-timely-executables.yml
with: # with:
tag: ${{ needs.set-tag.outputs.current_version }}-${{ needs.set-tag.outputs.nightly_build_tag }} # tag: ${{ needs.set-tag.outputs.current_version }}-${{ needs.set-tag.outputs.nightly_build_tag }}
secrets: # secrets:
NC_GITHUB_TOKEN: "${{ secrets.NC_GITHUB_TOKEN }}" # NC_GITHUB_TOKEN: "${{ secrets.NC_GITHUB_TOKEN }}"
# Build docker image and push to docker hub # Build docker image and push to docker hub
release-docker: release-docker:
needs: [set-tag, release-npm] needs: [set-tag]
uses: ./.github/workflows/release-docker.yml uses: ./.github/workflows/release-timely-docker.yml
with: with:
currentVersion: ${{ needs.set-tag.outputs.current_version }} currentVersion: ${{ needs.set-tag.outputs.current_version }}
tag: ${{ needs.set-tag.outputs.nightly_build_tag }} tag: ${{ needs.set-tag.outputs.nightly_build_tag }}

15
.github/workflows/release-pr.yml

@ -50,22 +50,11 @@ jobs:
target_tag: ${{ steps.tag-step.outputs.TARGET_TAG }} target_tag: ${{ steps.tag-step.outputs.TARGET_TAG }}
current_version: ${{ steps.tag-step.outputs.CURRENT_VERSION }} current_version: ${{ steps.tag-step.outputs.CURRENT_VERSION }}
# Build, install, publish frontend and backend to npm
release-npm:
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.event.pull_request.draft == false && github.base_ref == 'develop' && github.event.action != 'closed' }}
needs: [set-tag]
uses: ./.github/workflows/release-npm.yml
with:
tag: ${{ needs.set-tag.outputs.target_tag }}
targetEnv: 'DEV'
secrets:
NPM_TOKEN: "${{ secrets.NPM_TOKEN }}"
# Build docker image and push to docker hub # Build docker image and push to docker hub
release-docker: release-docker:
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.event.pull_request.draft == false && github.base_ref == 'develop' && github.event.action != 'closed' }} if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.event.pull_request.draft == false && github.base_ref == 'develop' && github.event.action != 'closed' }}
needs: [release-npm, set-tag] needs: [set-tag]
uses: ./.github/workflows/release-docker.yml uses: ./.github/workflows/release-timely-docker.yml
with: with:
currentVersion: ${{ needs.set-tag.outputs.current_version }} currentVersion: ${{ needs.set-tag.outputs.current_version }}
tag: ${{ needs.set-tag.outputs.target_tag }} tag: ${{ needs.set-tag.outputs.target_tag }}

161
.github/workflows/release-timely-docker.yml

@ -0,0 +1,161 @@
name: "Release : Docker"
on:
# Triggered manually
workflow_dispatch:
inputs:
tag:
description: "Docker image tag"
required: true
targetEnv:
description: "Target Environment"
required: true
type: choice
options:
- DEV
- PROD
# Triggered by release-nocodb.yml / release-nightly-dev.yml / release-pr.yml
workflow_call:
inputs:
tag:
description: "Docker image tag"
required: true
type: string
targetEnv:
description: "Target Environment"
required: true
type: string
isDaily:
description: "Is it triggered by daily schedule"
required: false
type: string
currentVersion:
description: "The current NocoDB version"
required: false
type: string
secrets:
DOCKERHUB_USERNAME:
required: true
DOCKERHUB_TOKEN:
required: true
jobs:
buildx:
runs-on: ubuntu-latest
env:
working-directory: ./packages/nocodb
steps:
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 9
- name: Get Docker Repository
id: get-docker-repository
run: |
DOCKER_REPOSITORY=nocodb-daily
DOCKER_BUILD_TAG=${{ github.event.inputs.tag || inputs.tag }}
DOCKER_BUILD_LATEST_TAG=latest
if [[ "$DOCKER_BUILD_TAG" =~ "-beta." ]]; then
DOCKER_BUILD_LATEST_TAG=$(echo $DOCKER_BUILD_TAG | awk -F '-beta.' '{print $1}')-beta.latest
fi
if [[ ${{ github.event.inputs.targetEnv || inputs.targetEnv }} == 'DEV' ]]; then
if [[ ${{ github.event.inputs.currentVersion || inputs.currentVersion || 'N/A' }} != 'N/A' ]]; then
DOCKER_BUILD_TAG=${{ github.event.inputs.currentVersion || inputs.currentVersion }}-${{ github.event.inputs.tag || inputs.tag }}
fi
if [[ ${{ inputs.isDaily || 'N' }} == 'Y' ]]; then
DOCKER_REPOSITORY=${DOCKER_REPOSITORY}-daily
else
DOCKER_REPOSITORY=${DOCKER_REPOSITORY}-timely
fi
fi
echo "DOCKER_REPOSITORY=${DOCKER_REPOSITORY}" >> $GITHUB_OUTPUT
echo "DOCKER_BUILD_TAG=${DOCKER_BUILD_TAG}" >> $GITHUB_OUTPUT
echo "DOCKER_BUILD_LATEST_TAG=${DOCKER_BUILD_LATEST_TAG}" >> $GITHUB_OUTPUT
echo DOCKER_REPOSITORY: ${DOCKER_REPOSITORY}
echo DOCKER_BUILD_TAG: ${DOCKER_BUILD_TAG}
echo DOCKER_BUILD_LATEST_TAG: ${DOCKER_BUILD_LATEST_TAG}
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.ref }}
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: 18.19.1
- name: install dependencies
run: pnpm bootstrap
- name: Build gui and sdk
run: |
pnpm bootstrap &&
cd packages/nc-gui &&
pnpm run generate
# copy build to nocodb
rsync -rvzh ./dist/ ../nocodb/docker/nc-gui/
- name: build nocodb
run: |
# build nocodb ( pack nocodb-sdk and nc-gui )
cd packages/nocodb &&
EE=true pnpm exec webpack --config webpack.timely.config.js &&
# remove bundled libraries (nocodb-sdk, knex-snowflake)
pnpm uninstall --save-prod nocodb-sdk
- name: Update version in package.json
run: |
# update package.json
cd packages/nocodb &&
jq --arg VERSION "$VERSION" '.version = $VERSION' package.json > tmp.json &&
mv tmp.json package.json
env:
VERSION: ${{ steps.get-docker-repository.outputs.DOCKER_BUILD_TAG }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v2.1.0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2.2.1
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to DockerHub
uses: docker/login-action@v2.1.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v3.2.0
with:
context: ${{ env.working-directory }}
file: ${{ env.working-directory }}/Dockerfile.timely
build-args: NC_VERSION=${{ steps.get-docker-repository.outputs.DOCKER_BUILD_TAG }}
platforms: linux/amd64,linux/arm64,linux/arm/v7
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
push: true
tags: |
nocodb/${{ steps.get-docker-repository.outputs.DOCKER_REPOSITORY }}:${{ steps.get-docker-repository.outputs.DOCKER_BUILD_TAG }}
nocodb/${{ steps.get-docker-repository.outputs.DOCKER_REPOSITORY }}:${{ steps.get-docker-repository.outputs.DOCKER_BUILD_LATEST_TAG }}
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache

80
packages/nocodb/Dockerfile.timely

@ -0,0 +1,80 @@
# syntax=docker/dockerfile:1
###########
# Litestream Builder
###########
FROM golang:alpine3.19 as lt-builder
WORKDIR /usr/src/
RUN apk add --no-cache git make musl-dev gcc
# build litestream
RUN git clone https://github.com/benbjohnson/litestream.git litestream
RUN cd litestream && go install ./cmd/litestream
RUN cp $GOPATH/bin/litestream /usr/src/lt
###########
# Builder
###########
FROM node:18.19.1-alpine as builder
WORKDIR /usr/src/app
# install node-gyp dependencies
RUN apk add --no-cache python3 make g++
# install pnpm
RUN corepack enable && corepack prepare pnpm@latest --activate
# Copy application dependency manifests to the container image.
COPY --link ./package.json ./package.json
COPY --link ./docker/main.js ./docker/main.js
COPY --link ./docker/start-litestream.sh /usr/src/appEntry/start.sh
COPY --link src/public/ ./docker/public/
COPY --link ./docker/nc-gui/ ./docker/nc-gui/
# for pnpm to generate a flat node_modules without symlinks
# so that modclean could work as expected
RUN echo "node-linker=hoisted" > .npmrc
# install production dependencies,
# reduce node_module size with modclean & removing sqlite deps,
# and add execute permission to start.sh
RUN pnpm install --prod --shamefully-hoist \
&& pnpm dlx modclean --patterns="default:*" --ignore="nc-lib-gui/**,dayjs/**,express-status-monitor/**,@azure/msal-node/dist/**" --run \
&& rm -rf ./node_modules/sqlite3/deps \
&& chmod +x /usr/src/appEntry/start.sh
##########
# Runner
##########
FROM alpine:3.19
WORKDIR /usr/src/app
ENV LITESTREAM_S3_SKIP_VERIFY=false \
LITESTREAM_RETENTION=1440h \
LITESTREAM_RETENTION_CHECK_INTERVAL=72h \
LITESTREAM_SNAPSHOT_INTERVAL=24h \
LITESTREAM_SYNC_INTERVAL=60s \
NC_DOCKER=0.6 \
NC_TOOL_DIR=/usr/app/data/ \
NODE_ENV=production \
PORT=8080
RUN apk add --update --no-cache \
dasel \
dumb-init \
nodejs
# Copy litestream binary and config file
COPY --link --from=lt-builder /usr/src/lt /usr/local/bin/litestream
COPY --link ./docker/litestream.yml /etc/litestream.yml
# Copy production code & main entry file
COPY --link --from=builder /usr/src/app/ /usr/src/app/
COPY --link --from=builder /usr/src/appEntry/ /usr/src/appEntry/
EXPOSE 8080
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
# Start Nocodb
CMD ["/usr/src/appEntry/start.sh"]

21
packages/nocodb/src/run/timely.ts

@ -0,0 +1,21 @@
import path from 'path';
import cors from 'cors';
import express from 'express';
import Noco from '~/Noco';
const server = express();
server.enable('trust proxy');
server.use(cors());
server.use(
process.env.NC_DASHBOARD_URL ?? '/dashboard',
express.static(path.join(__dirname, 'nc-gui')),
);
server.set('view engine', 'ejs');
(async () => {
const httpServer = server.listen(process.env.PORT || 8080, async () => {
console.log(`App started successfully.\nVisit -> ${Noco.dashboardUrl}`);
server.use(await Noco.init({}, httpServer, server));
});
})().catch((e) => console.log(e));

7
packages/nocodb/src/services/utils.service.ts

@ -80,7 +80,7 @@ export class UtilsService {
constructor(protected readonly configService: ConfigService<AppConfig>) {} constructor(protected readonly configService: ConfigService<AppConfig>) {}
lastSyncTime = dayjs(); lastSyncTime = null;
async versionInfo() { async versionInfo() {
if ( if (
@ -526,7 +526,10 @@ export class UtilsService {
} }
let payload = null; let payload = null;
if (dayjs().isAfter(this.lastSyncTime.add(3, 'hours'))) { if (
!this.lastSyncTime ||
dayjs().isAfter(this.lastSyncTime.add(3, 'hours'))
) {
payload = await T.payload(); payload = await T.payload();
this.lastSyncTime = dayjs(); this.lastSyncTime = dayjs();
} }

2
packages/nocodb/src/utils/TeleBatchProcessor.ts

@ -43,7 +43,7 @@ class TeleBatchProcessor {
return; return;
} }
await axios.post('https://nocodb.com/api/v1/telemetry', batch); await axios.post('https://telemetry.nocodb.com/api/v1/telemetry', batch);
} }
} }

1
packages/nocodb/src/utils/tele.ts

@ -264,6 +264,7 @@ class Tele {
xc_version: process.env.NC_SERVER_UUID, xc_version: process.env.NC_SERVER_UUID,
env: process.env.NODE_ENV || 'production', env: process.env.NODE_ENV || 'production',
oneClick: !!process.env.NC_ONE_CLICK, oneClick: !!process.env.NC_ONE_CLICK,
disabled: isDisabled,
}; };
try { try {
payload.os_type = os.type(); payload.os_type = os.type();

57
packages/nocodb/webpack.timely.config.js

@ -0,0 +1,57 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const CopyPlugin = require('copy-webpack-plugin');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('./build-utils/resolveTsAliases');
module.exports = {
entry: './src/run/timely.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [new TerserPlugin()],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('./tsconfig.json')),
},
mode: 'production',
output: {
filename: 'main.js',
path: path.resolve(__dirname, 'docker'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new webpack.EnvironmentPlugin(['EE']),
new CopyPlugin({
patterns: [{ from: 'src/public', to: 'public' }],
}),
],
target: 'node',
};
Loading…
Cancel
Save