Browse Source

chore: migrate to rspack 🦀 (#9839)

* chore: build to rspack and swc

* wip: rspack

* fix: rspack with hmr for local dev

* fix: pino pretty in local dev

* fix: moved all local dev config

* fix: add missing dependencies

* fix: unit tests ci

* fix: add tsconfig type checking

* fix: ignore watch node_modules

* fix: rspack build

* feat: add type checker

* fix: swagger tests

* fix: gtihub actions

* fix: do not remove nocodb-sdk

* fix: build workflow

* fix: update rspack config

* fix: rspack config sync

* fix: rspack sync

* fix: update configs

* fix: build config

* fix: sync rspack config

* fix: bump rspack version

* fix: rspack local not working
pull/9964/head
Anbarasu 1 week ago committed by GitHub
parent
commit
bc50e73f2b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 2
      .github/workflows/release-timely-docker.yml
  2. 2
      build-local-docker-image.sh
  3. 80
      packages/nocodb/docker/rspack.config.js
  4. 51
      packages/nocodb/docker/webpack.config.js
  5. 43
      packages/nocodb/package.json
  6. 94
      packages/nocodb/rspack.cli.config.js
  7. 100
      packages/nocodb/rspack.config.js
  8. 105
      packages/nocodb/rspack.dev.config.js
  9. 94
      packages/nocodb/rspack.timely.config.js
  10. 7
      packages/nocodb/src/Noco.ts
  11. 4
      packages/nocodb/src/controllers/integrations.controller.ts
  12. 4
      packages/nocodb/src/controllers/org-users.controller.ts
  13. 14
      packages/nocodb/src/db/BaseModelSqlv2.ts
  14. 4
      packages/nocodb/src/db/sql-mgr/SqlMgr.ts
  15. 4
      packages/nocodb/src/db/sql-migrator/lib/templates/mssql.template.ts
  16. 6
      packages/nocodb/src/helpers/NcPluginMgrv2.ts
  17. 10
      packages/nocodb/src/interface/Jobs.ts
  18. 13
      packages/nocodb/src/models/Comment.ts
  19. 2
      packages/nocodb/src/models/Model.ts
  20. 92
      packages/nocodb/src/modules/jobs/jobs-map.service.ts
  21. 68
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  22. 4
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts
  23. 4
      packages/nocodb/src/plugins/gcs/Gcs.ts
  24. 4
      packages/nocodb/src/services/command-palette.service.ts
  25. 2
      packages/nocodb/src/services/public-datas.service.ts
  26. 1
      packages/nocodb/src/services/sql-views.service.ts
  27. 2
      packages/nocodb/src/services/tables.service.ts
  28. 2
      packages/nocodb/src/services/views.service.ts
  29. 3
      packages/nocodb/tests/unit/tsconfig.ee.json
  30. 3
      packages/nocodb/tests/unit/tsconfig.json
  31. 7
      packages/nocodb/tsconfig.json
  32. 61
      packages/nocodb/webpack.cli.config.js
  33. 60
      packages/nocodb/webpack.config.js
  34. 59
      packages/nocodb/webpack.local.config.js
  35. 57
      packages/nocodb/webpack.timely.config.js
  36. 2081
      pnpm-lock.yaml

2
.github/workflows/release-timely-docker.yml

@ -104,7 +104,7 @@ jobs:
run: |
# build nocodb ( pack nocodb-sdk and nc-gui )
cd packages/nocodb &&
EE=true pnpm exec webpack --config webpack.timely.config.js &&
EE=true pnpm exec rspack --config rspack.timely.config.js &&
# remove bundled libraries (nocodb-sdk, knex-snowflake)
pnpm uninstall --save-prod nocodb-sdk

2
build-local-docker-image.sh

@ -45,7 +45,7 @@ function copy_gui_artifacts() {
function package_nocodb() {
# build nocodb ( pack nocodb-sdk and nc-gui )
cd ${SCRIPT_DIR}/packages/nocodb
EE=true ${SCRIPT_DIR}/node_modules/.bin/webpack --config ${SCRIPT_DIR}/packages/nocodb/webpack.local.config.js || ERROR="package_nocodb failed"
EE=true ${SCRIPT_DIR}/node_modules/@rspack/cli/bin --config ${SCRIPT_DIR}/packages/nocodb/rspack.config.js || ERROR="package_nocodb failed"
}
function build_image() {

80
packages/nocodb/docker/rspack.config.js

@ -0,0 +1,80 @@
const path = require('path');
const { rspack } = require('@rspack/core');
const nodeExternals = require('webpack-node-externals');
module.exports = {
entry: './src/run/dockerEntry.ts',
module: {
rules: [
{
test: /\.node$/,
loader: 'node-loader',
options: {
name: '[path][name].[ext]',
},
},
{
test: /\.tsx?$/,
exclude: /node_modules/,
loader: 'builtin:swc-loader',
options: {
jsc: {
parser: {
syntax: 'typescript',
tsx: true,
decorators: true,
dynamicImport: true,
},
transform: {
legacyDecorator: true,
decoratorMetadata: true,
},
target: 'es2017',
},
module: {
type: 'commonjs',
},
},
},
],
},
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
tsConfig: {
configFile: path.resolve('tsconfig.json'),
},
},
output: {
path: path.resolve('./docker'),
filename: 'main.js',
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [
new rspack.SwcJsMinimizerRspackPlugin({
minimizerOptions: {
compress: {
keep_classnames: true,
},
},
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
plugins: [
new rspack.EnvironmentPlugin({
EE: true,
}),
],
target: 'node',
node: {
__dirname: false,
},
};

51
packages/nocodb/docker/webpack.config.js

@ -1,51 +0,0 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('../build-utils/resolveTsAliases');
module.exports = {
entry: './src/run/dockerEntry.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('tsconfig.json')),
},
output: {
path: require('path').resolve('./docker'),
filename: 'main.js',
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [
new TerserPlugin({
terserOptions: {
keep_classnames: true,
},
}),
],
nodeEnv: false,
},
externals: [nodeExternals()],
plugins: [new webpack.EnvironmentPlugin(['EE'])],
target: 'node',
node: {
__dirname: false,
},
};

43
packages/nocodb/package.json

@ -21,8 +21,8 @@
"license": "AGPL-3.0-or-later",
"scripts": {
"build": "pnpm run docker:build",
"build:obfuscate": "EE=true webpack --config webpack.config.js",
"build:cli:module": "EE=true webpack --config webpack.cli.config.js",
"build:obfuscate": "EE=true rspack --config rspack.config.js",
"build:cli:module": "EE=true rspack --config rspack.cli.config.js",
"obfuscate:build:publish": "pnpm run build:obfuscate && pnpm publish .",
"registerIntegrations": "node build-utils/registerIntegrations.js",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
@ -32,19 +32,19 @@
"test": "jest --runInBand --forceExit",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:debug": "node --inspect-brk -r @swc-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json",
"watch:run": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/docker --log-error --project tsconfig.json\"",
"watch:run:mysql": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunMysql --log-error --project tsconfig.json\"",
"watch:run:pg": "cross-env NC_DISABLE_TELE=true NODE_ENV=development EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunPG --log-error --project tsconfig.json\"",
"watch:run:playwright:mysql": "rm -f ./test_noco.db; cross-env DB_TYPE=mysql NC_DB=\"mysql2://localhost:3306?u=root&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"",
"watch:run:playwright:pg": "rm -f ./test_noco.db; cross-env DB_TYPE=pg NC_DB=\"pg://localhost:5432?u=postgres&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"",
"watch:run:playwright": "rm -f ./test_*.db; cross-env DB_TYPE=sqlite DATABASE_URL=sqlite:./test_noco.db PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true EE=true NC_SNAPSHOT_WINDOW_SEC=3 nodemon -e ts,js -w ./src -x \"ts-node src/run/testDocker --log-error --project tsconfig.json\"",
"watch:run:playwright:quick": "rm -f ./test_noco.db; cp ../../tests/playwright/fixtures/noco_0_91_7.db ./test_noco.db; cross-env NODE_ENV=test DATABASE_URL=sqlite:./test_noco.db NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/docker --log-error --project tsconfig.json\"",
"watch:run:playwright:pg:cyquick": "rm -f ./test_noco.db; cp ../../tests/playwright/fixtures/noco_0_91_7.db ./test_noco.db; cross-env NODE_ENV=test NC_DISABLE_TELE=true EE=true nodemon -e ts,js -w ./src -x \"ts-node src/run/dockerRunPG_CyQuick.ts --log-error --project tsconfig.json\"",
"test:unit": "cross-env EE=false NODE_ENV=test TS_NODE_PROJECT=./tests/unit/tsconfig.json mocha -r ts-node/register tests/unit/index.test.ts --recursive --timeout 300000 --exit --delay",
"test:unit:pg": "cp tests/unit/.pg.env tests/unit/.env; cross-env NODE_ENV=test EE=false TS_NODE_PROJECT=./tests/unit/tsconfig.json mocha -r ts-node/register tests/unit/index.test.ts --recursive --timeout 300000 --exit --delay",
"docker:build": "EE=\"true-xc-test\" webpack --config docker/webpack.config.js"
"watch:run": "cross-env NODE_ENV=development NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/docker",
"watch:run:mysql": "cross-env NODE_ENV=development NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/dockerRunMysql",
"watch:run:pg": "cross-env NODE_ENV=development NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/dockerRunPG",
"watch:run:playwright": "rm -f ./test_*.db; cross-env DB_TYPE=sqlite DATABASE_URL=sqlite:./test_noco.db PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true NC_SNAPSHOT_WINDOW_SEC=3 rspack serve --config rspack.dev.config.js --entry src/run/testDocker",
"watch:run:playwright:mysql": "rm -f ./test_noco.db; cross-env DB_TYPE=mysql NC_DB=\"mysql2://localhost:3306?u=root&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/testDocker",
"watch:run:playwright:pg": "rm -f ./test_noco.db; cross-env DB_TYPE=pg NC_DB=\"pg://localhost:5432?u=postgres&p=password&d=pw_ncdb\" PLAYWRIGHT_TEST=true NODE_ENV=test NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/testDocker",
"watch:run:playwright:quick": "rm -f ./test_noco.db; cp ../../tests/playwright/fixtures/noco_0_91_7.db ./test_noco.db; cross-env NODE_ENV=test DATABASE_URL=sqlite:./test_noco.db NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/docker",
"watch:run:playwright:pg:cyquick": "rm -f ./test_noco.db; cp ../../tests/playwright/fixtures/noco_0_91_7.db ./test_noco.db; cross-env NODE_ENV=test NC_DISABLE_TELE=true rspack serve --config rspack.dev.config.js --entry src/run/dockerRunPG_CyQuick.ts",
"test:unit": "cross-env NODE_ENV=test TS_NODE_PROJECT=./tests/unit/tsconfig.json mocha --require @swc-node/register tests/unit/index.test.ts --recursive --timeout 300000 --exit --delay",
"test:unit:pg": "cp tests/unit/.pg.env tests/unit/.env; cross-env NODE_ENV=test TS_NODE_PROJECT=./tests/unit/tsconfig.json mocha --require @swc-node/register tests/unit/index.test.ts --recursive --timeout 300000 --exit --delay",
"docker:build": "EE=\"true-xc-test\" rspack --config docker/rspack.config.js"
},
"dependencies": {
"@aws-sdk/client-kafka": "^3.620.0",
@ -165,6 +165,9 @@
"@nestjs/schematics": "^10.1.1",
"@nestjs/testing": "^10.3.8",
"@nestjsplus/dyn-schematics": "^1.0.12",
"@rspack/cli": "^1.1.5",
"@rspack/core": "^1.1.5",
"@swc-node/register": "^1.10.9",
"@types/content-disposition": "^0.5.8",
"@types/ejs": "^3.1.5",
"@types/express": "^4.17.21",
@ -178,7 +181,6 @@
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"chai": "^4.4.1",
"copy-webpack-plugin": "^11.0.0",
"cross-env": "^7.0.3",
"eslint": "^8.56.0",
"eslint-config-prettier": "^8.10.0",
@ -186,17 +188,14 @@
"eslint-plugin-functional": "^5.0.8",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-prettier": "^4.2.1",
"fork-ts-checker-webpack-plugin": "^9.0.2",
"jest": "29.7.0",
"mocha": "^10.3.0",
"nodemon": "^3.0.3",
"node-loader": "^2.1.0",
"prettier": "^2.8.8",
"source-map-support": "^0.5.21",
"run-script-webpack-plugin": "^0.2.0",
"supertest": "^6.3.4",
"ts-jest": "29.1.2",
"ts-loader": "^9.5.1",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.3.3",
"webpack-cli": "^5.1.4"
"typescript": "^5.3.3"
}
}

94
packages/nocodb/rspack.cli.config.js

@ -0,0 +1,94 @@
const { resolve, } = require('path');
const { rspack } = require('@rspack/core');
const nodeExternals = require('webpack-node-externals');
module.exports = {
entry: './src/cli.ts',
module: {
rules: [
{
test: /\.node$/,
loader: 'node-loader',
options: {
name: '[path][name].[ext]',
},
},
{
test: /\.tsx?$/,
exclude: /node_modules/,
loader: 'builtin:swc-loader',
options: {
sourceMap: false,
jsc: {
parser: {
syntax: 'typescript',
tsx: true,
decorators: true,
dynamicImport: true,
},
transform: {
legacyDecorator: true,
decoratorMetadata: true,
},
target: 'es2017',
loose: true,
externalHelpers: false,
keepClassNames: true,
},
module: {
type: 'commonjs',
strict: false,
strictMode: true,
lazy: false,
noInterop: false,
},
},
},
],
},
optimization: {
minimize: true,
minimizer: [
new rspack.SwcJsMinimizerRspackPlugin({
minimizerOptions: {
format: {
comments: false,
},
},
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
tsConfig: {
configFile: resolve('tsconfig.json'),
},
},
mode: 'production',
output: {
filename: 'cli.js',
path: resolve(__dirname, '..', 'nc-secret-mgr', 'src/nocodb'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new rspack.EnvironmentPlugin({
EE: true,
}),
new rspack.BannerPlugin({
banner: 'This is a generated file. Do not edit',
entryOnly: true,
}),
],
target: 'node',
};

100
packages/nocodb/rspack.config.js

@ -0,0 +1,100 @@
const { resolve } = require('path');
const { rspack } = require('@rspack/core');
const nodeExternals = require('webpack-node-externals');
const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin');
module.exports = {
entry: './src/index.ts',
module: {
rules: [
{
test: /\.node$/,
loader: 'node-loader',
options: {
name: '[path][name].[ext]',
},
},
{
test: /\.tsx?$/,
exclude: /node_modules/,
loader: 'builtin:swc-loader',
options: {
sourceMap: false,
jsc: {
parser: {
syntax: 'typescript',
tsx: true,
decorators: true,
dynamicImport: true,
},
transform: {
legacyDecorator: true,
decoratorMetadata: true,
},
target: 'es2017',
loose: true,
externalHelpers: false,
keepClassNames: true,
},
module: {
type: 'commonjs',
strict: false,
strictMode: true,
lazy: false,
noInterop: false,
},
},
},
],
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [
new rspack.SwcJsMinimizerRspackPlugin({
minimizerOptions: {
compress: {
keep_classnames: true,
},
},
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json', '.node'],
tsConfig: {
configFile: resolve('tsconfig.json'),
},
},
mode: 'production',
output: {
filename: 'bundle.js',
path: resolve(__dirname, 'dist'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new rspack.EnvironmentPlugin({
EE: true,
}),
new rspack.CopyRspackPlugin({
patterns: [{ from: 'src/public', to: 'public' }],
}),
new ForkTsCheckerWebpackPlugin({
typescript: {
configFile: resolve('tsconfig.json'),
},
})
],
target: 'node',
};

105
packages/nocodb/rspack.dev.config.js

@ -0,0 +1,105 @@
const { join, resolve } = require('path');
const { rspack } = require('@rspack/core');
const nodeExternals = require('webpack-node-externals');
const { RunScriptWebpackPlugin } = require('run-script-webpack-plugin');
const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin');
const baseDevConfig = {
mode: 'development',
target: 'node',
devtool: 'eval-source-map',
module: {
rules: [
{
test: /\.node$/,
loader: 'node-loader',
options: {
name: '[path][name].[ext]',
},
},
{
test: /\.tsx?$/,
exclude: /node_modules/,
loader: 'builtin:swc-loader',
options: {
sourceMap: true,
jsc: {
parser: {
syntax: 'typescript',
tsx: true,
decorators: true,
dynamicImport: true,
},
transform: {
legacyDecorator: true,
decoratorMetadata: true,
},
target: 'es2017',
loose: true,
externalHelpers: false,
keepClassNames: true,
},
module: {
type: 'commonjs',
strict: false,
strictMode: true,
lazy: false,
noInterop: false,
},
},
},
],
},
externals: [
nodeExternals({
allowlist: ['webpack/hot/poll?1000'],
}),
],
resolve: {
tsConfig: {
configFile: resolve('tsconfig.json'),
},
extensions: ['.tsx', '.ts', '.js', '.json', '.node'],
},
optimization: {
minimize: false,
nodeEnv: false,
},
plugins: [
new rspack.EnvironmentPlugin({
EE: true,
NODE_ENV: 'development',
}),
new RunScriptWebpackPlugin({
name: 'main.js',
}),
new rspack.CopyRspackPlugin({
patterns: [{ from: 'src/public', to: 'public' }],
}),
new ForkTsCheckerWebpackPlugin({
typescript: {
configFile: join('tsconfig.json'),
},
}),
],
output: {
path: join(__dirname, 'dist'),
filename: 'main.js',
library: {
type: 'commonjs2',
},
clean: true,
},
devServer: {
devMiddleware: {
writeToDisk: true,
},
port: 9001,
},
watchOptions: {
ignored: /node_modules/,
poll: true,
},
};
module.exports = baseDevConfig;

94
packages/nocodb/rspack.timely.config.js

@ -0,0 +1,94 @@
const path = require('path');
const { rspack } = require('@rspack/core');
const nodeExternals = require('webpack-node-externals');
module.exports = {
entry: './src/run/timely.ts',
module: {
rules: [
{
test: /\.node$/,
loader: 'node-loader',
options: {
name: '[path][name].[ext]',
},
},
{
test: /\.tsx?$/,
exclude: /node_modules/,
loader: 'builtin:swc-loader',
options: {
sourceMap: false,
jsc: {
parser: {
syntax: 'typescript',
tsx: true,
decorators: true,
dynamicImport: true,
},
transform: {
legacyDecorator: true,
decoratorMetadata: true,
},
target: 'es2017',
loose: true,
externalHelpers: false,
keepClassNames: true,
},
module: {
type: 'commonjs',
strict: false,
strictMode: true,
lazy: false,
noInterop: false,
},
},
},
],
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [
new rspack.SwcJsMinimizerRspackPlugin({
minimizerOptions: {
compress: {
keep_classnames: true,
},
},
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json', '.node'],
tsConfig: {
configFile: path.resolve('tsconfig.json'),
},
},
mode: 'production',
output: {
filename: 'main.js',
path: path.resolve(__dirname, 'docker'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new rspack.EnvironmentPlugin({
EE: true,
}),
new rspack.CopyRspackPlugin({
patterns: [{ from: 'src/public', to: 'public' }],
}),
],
target: 'node',
};

7
packages/nocodb/src/Noco.ts

@ -97,6 +97,8 @@ export default class Noco {
return (this.ee = false);
}
declare module: any;
static async init(param: any, httpServer: http.Server, server: Express) {
const nestApp = await NestFactory.create(AppModule, {
bufferLogs: true,
@ -104,6 +106,11 @@ export default class Noco {
this.initCustomLogger(nestApp);
nestApp.flushLogs();
if ((module as any).hot) {
(module as any).hot.accept();
(module as any).hot.dispose(() => nestApp.close());
}
try {
this.sharp = (await import('sharp')).default;
} catch {

4
packages/nocodb/src/controllers/integrations.controller.ts

@ -11,9 +11,11 @@ import {
UseGuards,
} from '@nestjs/common';
import { IntegrationReqType, IntegrationsType } from 'nocodb-sdk';
// This service is overwritten entirely in the cloud and does not extend there.
// As a result, it refers to services from OSS to avoid type mismatches.
import { IntegrationsService } from 'src/services/integrations.service';
import { GlobalGuard } from '~/guards/global/global.guard';
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
import { IntegrationsService } from '~/services/integrations.service';
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
import { TenantContext } from '~/decorators/tenant-context.decorator';
import { NcContext, NcRequest } from '~/interface/config';

4
packages/nocodb/src/controllers/org-users.controller.ts

@ -11,9 +11,11 @@ import {
UseGuards,
} from '@nestjs/common';
import { OrgUserRoles } from 'nocodb-sdk';
// This service is overwritten entirely in the cloud and does not extend there.
// As a result, it refers to services from OSS to avoid type mismatches.
import { OrgUsersService } from 'src/services/org-users.service';
import { GlobalGuard } from '~/guards/global/global.guard';
import { PagedResponseImpl } from '~/helpers/PagedResponse';
import { OrgUsersService } from '~/services/org-users.service';
import { User } from '~/models';
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';

14
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -6,7 +6,6 @@ import utc from 'dayjs/plugin/utc.js';
import timezone from 'dayjs/plugin/timezone';
import equal from 'fast-deep-equal';
import {
AppEvents,
AuditOperationSubTypes,
AuditOperationTypes,
ButtonActionsType,
@ -74,7 +73,6 @@ import { defaultLimitConfig } from '~/helpers/extractLimitAndOffset';
import generateLookupSelectQuery from '~/db/generateLookupSelectQuery';
import { getAliasGenerator } from '~/utils';
import applyAggregation from '~/db/aggregation';
import { extractMentions } from '~/utils/richTextHelper';
import { chunkArray } from '~/utils/tsUtils';
dayjs.extend(utc);
@ -5336,7 +5334,7 @@ class BaseModelSqlv2 {
if (pkValues !== 'N/A' && pkValues !== undefined) {
dataWithPks.push({ pk: pkValues, data });
} else {
await this.prepareNocoData(data, true, cookie)
await this.prepareNocoData(data, true, cookie);
// const insertObj = this.handleValidateBulkInsert(data, columns);
dataWithoutPks.push(data);
}
@ -5356,10 +5354,10 @@ class BaseModelSqlv2 {
for (const { pk, data } of dataWithPks) {
if (existingPkSet.has(pk)) {
await this.prepareNocoData(data, false, cookie)
await this.prepareNocoData(data, false, cookie);
toUpdate.push(data);
} else {
await this.prepareNocoData(data, true, cookie)
await this.prepareNocoData(data, true, cookie);
// const insertObj = this.handleValidateBulkInsert(data, columns);
toInsert.push(data);
}
@ -6501,9 +6499,9 @@ class BaseModelSqlv2 {
* */
public async handleRichTextMentions(
prevData,
newData: Record<string, any> | Array<Record<string, any>>,
req,
_prevData,
_newData: Record<string, any> | Array<Record<string, any>>,
_req,
) {
return;
}

4
packages/nocodb/src/db/sql-mgr/SqlMgr.ts

@ -15,8 +15,10 @@ import type SqliteClient from '~/db/sql-client/lib/sqlite/SqliteClient';
import { T } from '~/utils';
import Result from '~/db/util/Result';
import Debug from '~/db/util/Debug';
import KnexMigrator from '~/db/sql-migrator/lib/KnexMigrator';
import SqlClientFactory from '~/db/sql-client/lib/SqlClientFactory';
// @ts-expect-error
import KnexMigrator from '~/db/sql-migrator/lib/KnexMigrator';
// @ts-expect-error
import NcConnectionMgr from '~/utils/common/NcConnectionMgr';
const randomID = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 20);

4
packages/nocodb/src/db/sql-migrator/lib/templates/mssql.template.ts

@ -10,7 +10,7 @@ export default {
client: 'mssql',
connection: {
host: DOCKER_DB_HOST || 'localhost',
port: DOCKER_DB_PORT ? parseInt(DOCKER_DB_PORT, 10) : null || 1433,
port: DOCKER_DB_PORT ? parseInt(DOCKER_DB_PORT, 10) : 1433,
user: 'sa',
password: 'Password123.',
database: 'default_dev',
@ -29,7 +29,7 @@ export default {
client: 'mssql',
connection: {
host: DOCKER_DB_HOST || 'localhost',
port: DOCKER_DB_PORT ? parseInt(DOCKER_DB_PORT) : null || 1433,
port: DOCKER_DB_PORT ? parseInt(DOCKER_DB_PORT) : 1433,
user: 'sa',
password: 'Password123.',
database: 'default_test',

6
packages/nocodb/src/helpers/NcPluginMgrv2.ts

@ -130,6 +130,9 @@ class NcPluginMgrv2 {
/*
* NC_S3_BUCKET_NAME
* NC_S3_REGION
* NC_S3_ENDPOINT
* NC_S3_ACCESS_KEY
* NC_S3_ACCESS_SECRET
* */
if (
@ -137,6 +140,7 @@ class NcPluginMgrv2 {
(process.env.NC_S3_REGION || process.env.NC_S3_ENDPOINT)
) {
const s3Plugin = await Plugin.getPlugin(S3PluginConfig.id);
const s3CfgData: Record<string, any> = {
bucket: process.env.NC_S3_BUCKET_NAME,
region: process.env.NC_S3_REGION,
@ -144,10 +148,12 @@ class NcPluginMgrv2 {
force_path_style: process.env.NC_S3_FORCE_PATH_STYLE === 'true',
acl: process.env.NC_S3_ACL,
};
if (process.env.NC_S3_ACCESS_KEY && process.env.NC_S3_ACCESS_SECRET) {
s3CfgData.access_key = process.env.NC_S3_ACCESS_KEY;
s3CfgData.access_secret = process.env.NC_S3_ACCESS_SECRET;
}
await Plugin.update(s3Plugin.id, {
active: true,
input: JSON.stringify(s3CfgData),

10
packages/nocodb/src/interface/Jobs.ts

@ -1,4 +1,10 @@
import type { AttachmentResType, PublicAttachmentScope, SupportedExportCharset, UserType, SnapshotType } from 'nocodb-sdk';
import type {
AttachmentResType,
PublicAttachmentScope,
SnapshotType,
SupportedExportCharset,
UserType,
} from 'nocodb-sdk';
import type { NcContext, NcRequest } from '~/interface/config';
export const JOBS_QUEUE = 'jobs';
@ -183,7 +189,7 @@ export interface RestoreSnapshotJobData extends JobData {
targetContext: {
workspace_id: string;
base_id: string;
}
};
snapshot: SnapshotType;
req: NcRequest;
}

13
packages/nocodb/src/models/Comment.ts

@ -56,20 +56,17 @@ export default class Comment implements CommentType {
fk_model_id,
},
orderBy: {
id: 'asc'
id: 'asc',
},
limit: pagination?.limit,
offset: pagination?.offset,
xcCondition: {
_or: [
{ is_deleted: { eq: null } },
{ is_deleted: { eq: true } },
]
}
}
_or: [{ is_deleted: { eq: null } }, { is_deleted: { eq: true } }],
},
},
);
return comments.map(comment => new Comment(comment));
return comments.map((comment) => new Comment(comment));
}
public static async list(

2
packages/nocodb/src/models/Model.ts

@ -1162,7 +1162,7 @@ export default class Model implements TableType {
context: NcContext,
{
modelId,
userId,
userId: _,
}: {
modelId: string;
userId?: string;

92
packages/nocodb/src/modules/jobs/jobs-map.service.ts

@ -28,55 +28,57 @@ export class JobsMap {
protected readonly useWorkerProcessor: UseWorkerProcessor,
) {}
protected _jobMap: {
protected get _jobMap(): {
[key in JobTypes]?: {
this: any;
fn?: string;
};
} = {
[JobTypes.DuplicateBase]: {
this: this.duplicateProcessor,
fn: 'duplicateBase',
},
[JobTypes.DuplicateModel]: {
this: this.duplicateProcessor,
fn: 'duplicateModel',
},
[JobTypes.DuplicateColumn]: {
this: this.duplicateProcessor,
fn: 'duplicateColumn',
},
[JobTypes.AtImport]: {
this: this.atImportProcessor,
},
[JobTypes.MetaSync]: {
this: this.metaSyncProcessor,
},
[JobTypes.SourceCreate]: {
this: this.sourceCreateProcessor,
},
[JobTypes.SourceDelete]: {
this: this.sourceDeleteProcessor,
},
[JobTypes.HandleWebhook]: {
this: this.webhookHandlerProcessor,
},
[JobTypes.DataExport]: {
this: this.dataExportProcessor,
},
[JobTypes.ThumbnailGenerator]: {
this: this.thumbnailGeneratorProcessor,
},
[JobTypes.AttachmentCleanUp]: {
this: this.attachmentCleanUpProcessor,
},
[JobTypes.InitMigrationJobs]: {
this: this.initMigrationJobs,
},
[JobTypes.UseWorker]: {
this: this.useWorkerProcessor,
},
};
} {
return {
[JobTypes.DuplicateBase]: {
this: this.duplicateProcessor,
fn: 'duplicateBase',
},
[JobTypes.DuplicateModel]: {
this: this.duplicateProcessor,
fn: 'duplicateModel',
},
[JobTypes.DuplicateColumn]: {
this: this.duplicateProcessor,
fn: 'duplicateColumn',
},
[JobTypes.AtImport]: {
this: this.atImportProcessor,
},
[JobTypes.MetaSync]: {
this: this.metaSyncProcessor,
},
[JobTypes.SourceCreate]: {
this: this.sourceCreateProcessor,
},
[JobTypes.SourceDelete]: {
this: this.sourceDeleteProcessor,
},
[JobTypes.HandleWebhook]: {
this: this.webhookHandlerProcessor,
},
[JobTypes.DataExport]: {
this: this.dataExportProcessor,
},
[JobTypes.ThumbnailGenerator]: {
this: this.thumbnailGeneratorProcessor,
},
[JobTypes.AttachmentCleanUp]: {
this: this.attachmentCleanUpProcessor,
},
[JobTypes.InitMigrationJobs]: {
this: this.initMigrationJobs,
},
[JobTypes.UseWorker]: {
this: this.useWorkerProcessor,
},
};
}
public get jobs() {
return this._jobMap;

68
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -35,42 +35,39 @@ export class DuplicateProcessor {
private readonly columnsService: ColumnsService,
) {}
async duplicateBaseJob(
{
sourceBase,
targetBase,
dataSource,
req,
context,
options,
operation,
targetContext: _targetContext
}: {
sourceBase: Base; // Base to duplicate
targetBase: Base; // Base to duplicate to
dataSource: Source; // Data source to duplicate from
req: NcRequest;
context: NcContext // Context of the base to duplicate
targetContext?: NcContext // Context of the base to duplicate to
options: {
excludeData?: boolean;
excludeHooks?: boolean;
excludeViews?: boolean;
excludeComments?: boolean;
}
operation: string
}) {
async duplicateBaseJob({
sourceBase,
targetBase,
dataSource,
req,
context,
options,
operation,
targetContext: _targetContext,
}: {
sourceBase: Base; // Base to duplicate
targetBase: Base; // Base to duplicate to
dataSource: Source; // Data source to duplicate from
req: NcRequest;
context: NcContext; // Context of the base to duplicate
targetContext?: NcContext; // Context of the base to duplicate to
options: {
excludeData?: boolean;
excludeHooks?: boolean;
excludeViews?: boolean;
excludeComments?: boolean;
};
operation: string;
}) {
const hrTime = initTime();
const targetContext = _targetContext ?? {
const targetContext = _targetContext ?? {
workspace_id: targetBase.fk_workspace_id,
base_id: targetBase.id,
}
};
try {
if(!sourceBase || !targetBase || !dataSource) {
if (!sourceBase || !targetBase || !dataSource) {
throw new Error(`Base or source not found!`);
}
@ -82,7 +79,7 @@ export class DuplicateProcessor {
const exportedModels = await this.exportService.serializeModels(context, {
modelIds: models.map((m) => m.id),
...options
...options,
});
elapsedTime(
@ -133,8 +130,7 @@ export class DuplicateProcessor {
user: req.user,
req,
});
} catch(err) {
} catch (err) {
if (targetBase?.id) {
await this.projectsService.baseSoftDelete(targetContext, {
baseId: targetBase.id,
@ -177,10 +173,10 @@ export class DuplicateProcessor {
excludeData,
excludeHooks,
excludeViews,
excludeComments
excludeComments,
},
operation: JobTypes.DuplicateBase
})
operation: JobTypes.DuplicateBase,
});
return { id: dupProject.id };
}

4
packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -384,13 +384,13 @@ export class ExportService {
if (!excludeComments) {
const READ_BATCH_SIZE = 100;
let comments: Comment[] = [];
const comments: Comment[] = [];
let offset = 0;
while (true) {
const batchComments = await Comment.listByModel(context, model.id, {
limit: READ_BATCH_SIZE + 1,
offset
offset,
});
comments.push(...batchComments.slice(0, READ_BATCH_SIZE));

4
packages/nocodb/src/plugins/gcs/Gcs.ts

@ -35,7 +35,7 @@ export default class Gcs implements IStorageAdapterV2 {
return patchedKey;
}
private aclConfig(): { predefinedAcl: 'publicRead' } | {} {
private aclConfig(): { predefinedAcl: 'publicRead' } | Record<string, never> {
return this.input.uniform_bucket_level_access
? {}
: { predefinedAcl: 'publicRead' };
@ -180,7 +180,7 @@ export default class Gcs implements IStorageAdapterV2 {
action: 'read',
expires: Date.now() + expiresInSeconds * 1000,
responseDisposition: pathParameters?.ResponseContentDisposition,
responseType: pathParameters?.ResponseContentType
responseType: pathParameters?.ResponseContentType,
};
const [url] = await this.storageClient

4
packages/nocodb/src/services/command-palette.service.ts

@ -1,7 +1,9 @@
import { Injectable, Logger } from '@nestjs/common';
import { type UserType, viewTypeAlias } from 'nocodb-sdk';
import { getCommandPaletteForUserWorkspace } from 'src/helpers/commandPaletteHelpers';
import { deserializeJSON } from '~/utils/serialize';
import { getCommandPaletteForUserWorkspace } from '~/helpers/commandPaletteHelpers';
// This service is overwritten entirely in the cloud and does not extend there.
// As a result, it refers to services from OSS to avoid type mismatches.
@Injectable()
export class CommandPaletteService {

2
packages/nocodb/src/services/public-datas.service.ts

@ -108,7 +108,7 @@ export class PublicDatasService {
query: any;
},
) {
const { sharedViewUuid, password, query = {} } = param;
const { sharedViewUuid, password } = param;
const view = await View.getByUUID(context, sharedViewUuid);
if (!view) NcError.viewNotFound(sharedViewUuid);

1
packages/nocodb/src/services/sql-views.service.ts

@ -147,6 +147,7 @@ export class SqlViewsService {
title: getTableNameAlias(body.view_name, base.prefix, source),
type: ModelTypes.VIEW,
order: +(tables?.pop()?.order ?? 0) + 1,
user_id: param.user.id,
});
let colOrder = 1;

2
packages/nocodb/src/services/tables.service.ts

@ -10,9 +10,7 @@ import {
ProjectRoles,
RelationTypes,
UITypes,
ViewLockType,
} from 'nocodb-sdk';
import { LockType } from 'nc-gui/lib/enums';
import { MetaDiffsService } from './meta-diffs.service';
import { ColumnsService } from './columns.service';
import type {

2
packages/nocodb/src/services/views.service.ts

@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
import { AppEvents, ProjectRoles, ViewLockType } from 'nocodb-sdk';
import { AppEvents, ProjectRoles } from 'nocodb-sdk';
import type {
SharedViewReqType,
UserType,

3
packages/nocodb/tests/unit/tsconfig.ee.json

@ -65,7 +65,8 @@
},
"include": [
"./tests/**/**/**.ts",
"./tests/**/**.ts"
"./tests/**/**.ts",
"../../src/**/**.ts",
// "**/*.ts",
// "**/*.json"
],

3
packages/nocodb/tests/unit/tsconfig.json

@ -60,7 +60,8 @@
},
"include": [
"./tests/**/**/**.ts",
"./tests/**/**.ts"
"./tests/**/**.ts",
"../../src/**/**.ts",
// "**/*.ts",
// "**/*.json"
],

7
packages/nocodb/tsconfig.json

@ -1,9 +1,4 @@
{
"ts-node": {
"require": [
"tsconfig-paths/register"
]
},
"compilerOptions": {
"module": "commonjs",
"declaration": true,
@ -41,5 +36,5 @@
]
},
"include": ["src/**/*"],
"exclude": ["src/ee", "src/ee-on-prem", "src/ee-cloud"]
"exclude": ["src/ee", "src/ee-on-prem", "src/ee-cloud", "**/*.spec.ts"]
}

61
packages/nocodb/webpack.cli.config.js

@ -1,61 +0,0 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('./build-utils/resolveTsAliases');
module.exports = {
entry: './src/cli.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
optimization: {
minimize: true,
minimizer: [
new TerserPlugin({
extractComments: false,
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('tsconfig.json')),
},
mode: 'production',
output: {
filename: 'cli.js',
path: path.resolve(__dirname, '..', 'nc-secret-mgr', 'src/nocodb'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new webpack.EnvironmentPlugin(['EE']),
new webpack.BannerPlugin({
banner: 'This is a generated file. Do not edit',
entryOnly:true
}),
],
target: 'node',
};

60
packages/nocodb/webpack.config.js

@ -1,60 +0,0 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const CopyPlugin = require('copy-webpack-plugin');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('./build-utils/resolveTsAliases');
module.exports = {
entry: './src/index.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [
new TerserPlugin({
terserOptions: {
keep_classnames: true,
},
}),
],
nodeEnv: false,
},
externals: [nodeExternals()],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('tsconfig.json')),
},
mode: 'production',
output: {
filename: 'bundle.js',
path: path.resolve(__dirname, 'dist'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new webpack.EnvironmentPlugin(['EE']),
new CopyPlugin({
patterns: [{ from: 'src/public', to: 'public' }],
}),
],
target: 'node',
};

59
packages/nocodb/webpack.local.config.js

@ -1,59 +0,0 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('./build-utils/resolveTsAliases');
module.exports = {
entry: './src/run/local.ts',
// devtool: 'inline-source-map',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [
new TerserPlugin({
terserOptions: {
keep_classnames: true,
},
}),
],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('tsconfig.json')),
},
mode: 'production',
output: {
filename: 'main.js',
path: path.resolve(__dirname, 'docker'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [new webpack.EnvironmentPlugin(['EE'])],
target: 'node',
};

57
packages/nocodb/webpack.timely.config.js

@ -1,57 +0,0 @@
const path = require('path');
const nodeExternals = require('webpack-node-externals');
const webpack = require('webpack');
const CopyPlugin = require('copy-webpack-plugin');
const TerserPlugin = require('terser-webpack-plugin');
const { resolveTsAliases } = require('./build-utils/resolveTsAliases');
module.exports = {
entry: './src/run/timely.ts',
module: {
rules: [
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: {
loader: 'ts-loader',
options: {
transpileOnly: true,
},
},
},
],
},
optimization: {
minimize: true, //Update this to true or false
minimizer: [new TerserPlugin()],
nodeEnv: false,
},
externals: [
nodeExternals({
allowlist: ['nocodb-sdk'],
}),
],
resolve: {
extensions: ['.tsx', '.ts', '.js', '.json'],
alias: resolveTsAliases(path.resolve('./tsconfig.json')),
},
mode: 'production',
output: {
filename: 'main.js',
path: path.resolve(__dirname, 'docker'),
library: 'libs',
libraryTarget: 'umd',
globalObject: "typeof self !== 'undefined' ? self : this",
},
node: {
__dirname: false,
},
plugins: [
new webpack.EnvironmentPlugin(['EE']),
new CopyPlugin({
patterns: [{ from: 'src/public', to: 'public' }],
}),
],
target: 'node',
};

2081
pnpm-lock.yaml

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save