github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

CI/CD improvements and archive files on shared system (#6469) 

* archive files on shared system and remove parallel build, 3 shards

* add commented lines back in the file as it will be needed
pull/6487/head
starbirdtech383 9 months ago committed by GitHub
parent
8227926720
commit
bf931cf63d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 78 additions and 103 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 27
      .github/workflows/ci-cd.yml
  2. 81
      .github/workflows/playwright-test-workflow.yml
  3. 62
      .github/workflows/pre-build-for-playwright.yml
  4. 11
      tests/playwright/package.json

27
.github/workflows/ci-cd.yml
Unescape View File

@ -118,19 +118,12 @@ jobs:
db: mysql
shard: 2
playwright-mysql-3:
needs: playwright-mysql-1
needs: pre-build-for-playwright
if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft )}}
uses: ./.github/workflows/playwright-test-workflow.yml
with:
db: mysql
shard: 3
playwright-mysql-4:
needs: playwright-mysql-2
if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft )}}
uses: ./.github/workflows/playwright-test-workflow.yml
with:
db: mysql
shard: 4
playwright-sqlite-1:
needs: pre-build-for-playwright
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }}
@ -145,6 +138,13 @@ jobs:
with:
db: sqlite
shard: 2
playwright-sqlite-3:
needs: pre-build-for-playwright
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }}
uses: ./.github/workflows/playwright-test-workflow.yml
with:
db: sqlite
shard: 3
playwright-pg-shard-1:
needs: pre-build-for-playwright
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }}
@ -160,16 +160,9 @@ jobs:
db: pg
shard: 2
playwright-pg-shard-3:
needs: playwright-pg-shard-1
needs: pre-build-for-playwright
if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft )}}
uses: ./.github/workflows/playwright-test-workflow.yml
with:
db: pg
shard: 3
playwright-pg-shard-4:
needs: playwright-pg-shard-2
if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft ) }}
uses: ./.github/workflows/playwright-test-workflow.yml
with:
db: pg
shard: 4
shard: 3

81
.github/workflows/playwright-test-workflow.yml
Unescape View File

@ -13,37 +13,16 @@ on:
jobs:
playwright:
runs-on: [self-hosted, v2]
runs-on: [self-hosted, v3]
timeout-minutes: 100
steps:
- name: Checkout
uses: actions/checkout@v3
- name: pre-requisite checks
shell: bash
run: |
node_version=$(node --version || echo "error")
pnpm_version=$(pnpm -v || echo "error")
echo "node version: $node_version"
echo "pnpm version: $pnpm_version"
if [[ $node_version != v18* ]] || [[ $pnp_version != 8* ]]; then
echo "version mismatch: expected node v18 and pnpm v8"
RUN_PRERQUISITE_STEPS="true"
elif [[ ${FORCE_RUN_PRERQUISITE_STEPS} == "true" ]];
# || [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
then
echo "FORCE_RUN_PRERQUISITE_STEPS is true"
RUN_PRERQUISITE_STEPS="true"
else
RUN_PRERQUISITE_STEPS="false"
fi
echo "RUN_PRERQUISITE_STEPS=${RUN_PRERQUISITE_STEPS}" >> $GITHUB_ENV
uses: actions/checkout@v3
- name: Setup Node
if: env.RUN_PRERQUISITE_STEPS == 'true'
uses: actions/setup-node@v3
with:
node-version: 18.14.0
- name: Setup pnpm
if: env.RUN_PRERQUISITE_STEPS == 'true'
uses: pnpm/action-setup@v2
with:
version: 8
@ -51,8 +30,21 @@ jobs:
shell: bash
run: |
echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
- name: check if npm cache is needed
shell: bash
run: |
IS_NPM_CACHE_DOWNLOAD_REQUIRED="/cache-marker-v1.txt"
# update the above file name to force the cache ex: /cache-marker-v2.txt.
if [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
then
echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED is true"
IS_NPM_CACHE_DOWNLOAD_REQUIRED="true"
else
IS_NPM_CACHE_DOWNLOAD_REQUIRED="false"
fi
echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED=${IS_NPM_CACHE_DOWNLOAD_REQUIRED}" >> $GITHUB_ENV
- uses: actions/cache@v3
if: env.RUN_PRERQUISITE_STEPS == 'true'
if: env.IS_NPM_CACHE_DOWNLOAD_REQUIRED == 'true'
name: Setup pnpm cache
with:
path: ${{ env.STORE_PATH }}
@ -113,22 +105,28 @@ jobs:
working-directory: ./packages/nocodb
run: |
pnpm run watch:run:playwright:pg &> ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
- name: copy built output and start frontend
if: always()
working-directory: ./
- name: download ui build from artf server
working-directory: ./packages/nc-gui/
run: |
# expects the variables to be available in runner context.
path="gh-artifacts/runs/${GITHUB_RUN_ID}/ui-build/.output"
target_dir="/mnt/${path}"
mkdir -p ${target_dir}
if [[ -d ${target_dir} ]]; then
echo "Directory ${target_dir} exists."
cp -r ${target_dir} ./packages/nc-gui/ || echo "playwright reports directory does not exists" >> ${target_dir}/playwright-report/index.html
else
echo "Error: Directory ${target_dir} does not exists."
exit 1
fi
cd ./packages/nc-gui/
rm -rf .output.zip .output
FILE="$(echo ${GITHUB_REPOSITORY} | sed "s,/,-,g")-${GITHUB_RUN_ID}.zip"
BUILD_FILE_URL=http://65.21.27.147/gh-artifacts/ui-builds/${FILE}
time curl -o output.zip ${BUILD_FILE_URL} -n
unzip -oq output.zip
# path="gh-artifacts/runs/${GITHUB_RUN_ID}/ui-build/.output"
# target_dir="/mnt/${path}"
# if [[ -d ${target_dir} ]]; then
# echo "Directory ${target_dir} exists."
# cp -r ${target_dir} . || echo "playwright reports directory does not exists" >> ${target_dir}/playwright-report/index.html
# else
# echo "Error: Directory ${target_dir} does not exists."
# exit 1
# fi
- name: start frontend
working-directory: ./packages/nc-gui/
run: |
pnpm run ci:start
- name: Install Playwright Browsers
working-directory: ./tests/playwright
@ -183,7 +181,8 @@ jobs:
# - name: Run quick tests
# if: ${{ inputs.db == 'sqlite' }}
# working-directory: ./tests/playwright
# run: PLAYWRIGHT_HTML_REPORT=playwright-report-quick npm run test:quick
# run: PLAYWRIGHT_HTML_REPORT=playwright-report-quick npm run test:quick
- uses: actions/upload-artifact@v3
if: ${{ inputs.db == 'sqlite' }}
with:
@ -234,6 +233,6 @@ jobs:
cp -r ./tests/playwright/playwright-report ${target_dir}/ || echo "playwright reports directory does not exists" >> ${target_dir}/playwright-report/index.html
cp ./packages/nocodb/*_test_backend.log ${target_dir}/ || echo "backend logs file does not exists" >> ${target_dir}/index.html
# end: artifacts copy
SUMMARY='[Artifacts](http://135.181.48.96/'${path}')
[playwright-report](http://135.181.48.96/'${path}'/playwright-report)'
SUMMARY='[Artifacts]('${REPORTS_HOST}/${path}')
[playwright-report]('${REPORTS_HOST}/${path}'/playwright-report)'
echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY

62
.github/workflows/pre-build-for-playwright.yml
Unescape View File

@ -11,46 +11,38 @@ on:
jobs:
playwright:
runs-on: [self-hosted, aws]
runs-on: [self-hosted, v3]
timeout-minutes: 100
steps:
- name: Checkout
uses: actions/checkout@v3
- name: pre-requisite checks
shell: bash
run: |
node_version=$(node --version || echo "error")
pnpm_version=$(pnpm -v || echo "error")
echo "node version: $node_version"
echo "pnpm version: $pnpm_version"
if [[ $node_version != v18* ]] || [[ $pnp_version != 8* ]]; then
echo "version mismatch: expected node v18 and pnpm v8"
RUN_PRERQUISITE_STEPS="true"
elif [[ ${FORCE_RUN_PRERQUISITE_STEPS} == "true" ]];
# || [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
then
echo "FORCE_RUN_PRERQUISITE_STEPS is true"
RUN_PRERQUISITE_STEPS="true"
else
RUN_PRERQUISITE_STEPS="false"
fi
echo "RUN_PRERQUISITE_STEPS=${RUN_PRERQUISITE_STEPS}" >> $GITHUB_ENV
- name: Setup Node
if: env.RUN_PRERQUISITE_STEPS == 'true'
uses: actions/setup-node@v3
with:
node-version: 18.14.0
- name: Setup pnpm
if: env.RUN_PRERQUISITE_STEPS == 'true'
uses: pnpm/action-setup@v2
with:
version: 8
version: 8
- name: check if npm cache is needed
shell: bash
run: |
PRE_REQ_CHECK_FILE_PATH="/cache-marker-v1.txt"
# update the above file name to force the cache ex: /cache-marker-v2.txt.
if [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
then
echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED is true"
IS_NPM_CACHE_DOWNLOAD_REQUIRED="true"
else
IS_NPM_CACHE_DOWNLOAD_REQUIRED="false"
fi
echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED=${IS_NPM_CACHE_DOWNLOAD_REQUIRED}" >> $GITHUB_ENV
- name: Get pnpm store directory
shell: bash
run: |
echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
echo "STORE_PATH=/root/setup-pnpm/node_modules/.bin/store/v3" >> $GITHUB_ENV
- uses: actions/cache@v3
if: env.RUN_PRERQUISITE_STEPS == 'true'
if: env.IS_NPM_CACHE_DOWNLOAD_REQUIRED == 'true'
name: Setup pnpm cache
with:
path: ${{ env.STORE_PATH }}
@ -59,22 +51,18 @@ jobs:
${{ runner.os }}-pnpm-store-
- name: install dependencies
run: pnpm bootstrap
- name: run frontend
- name: build frontend (nc-gui)
working-directory: ./packages/nc-gui
run: |
pnpm run build
timeout-minutes: 20
- name: Copy Artifacts to Local Artifacts Dir
if: always()
- name: upload frontend (nc-gui) build to artf server
working-directory: ./
run: |
# expects the variables to be available in runner context.
path="gh-artifacts/runs/${GITHUB_RUN_ID}/ui-build/"
target_dir="/mnt/${path}"
mkdir -p ${target_dir}
# start : add any artifacts to be copied here
cp -r ./packages/nc-gui/.output ${target_dir}/ || echo "playwright reports directory does not exists" >> ${target_dir}/playwright-report/index.html
SUMMARY='[UI BUILD](http://65.21.27.147/'${path}')'
echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY
FILE="$(echo ${GITHUB_REPOSITORY} | sed "s,/,-,g")-${GITHUB_RUN_ID}.zip"
cd ./packages/nc-gui/
zip -r ${FILE} .output || echo "UI build directory does not exists" >&2
echo "uploading ${FILE} to http://65.21.27.147/upload/${FILE}"
time curl -T "${FILE}" http://65.21.27.147/upload/${FILE} -n

11
tests/playwright/package.json
Unescape View File

@ -23,10 +23,6 @@
"scripts": {
"test": "TRACE=true pnpm exec playwright test --workers=4",
"test:fast": "pnpm exec playwright test --workers=6",
"test:shard:1": "TRACE=true pnpm exec playwright test --workers=4 --shard=1/4",
"test:shard:2": "TRACE=true pnpm exec playwright test --workers=4 --shard=2/4",
"test:shard:3": "TRACE=true pnpm exec playwright test --workers=4 --shard=3/4",
"test:shard:4": "TRACE=true pnpm exec playwright test --workers=4 --shard=4/4",
"test:repeat": "TRACE=true pnpm exec playwright test --workers=4 --repeat-each=12",
"test:quick": "TRACE=true PW_QUICK_TEST=1 pnpm exec playwright test --workers=4",
"test:debug": "./startPlayWrightServer.sh; PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 0 --workers 1 --max-failures=1",
@ -34,10 +30,9 @@
"test:debug:watch": "pnpm dlx nodemon -e ts -w ./ -x \"pnpm run test:debug\"",
"test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1",
"ci:test": "pnpm exec playwright test --workers=2",
"ci:test:shard:1": "pnpm exec playwright test --workers=4 --shard=1/4",
"ci:test:shard:2": "pnpm exec playwright test --workers=4 --shard=2/4",
"ci:test:shard:3": "pnpm exec playwright test --workers=4 --shard=3/4",
"ci:test:shard:4": "pnpm exec playwright test --workers=4 --shard=4/4",
"ci:test:shard:1": "pnpm exec playwright test --workers=4 --shard=1/3",
"ci:test:shard:2": "pnpm exec playwright test --workers=4 --shard=2/3",
"ci:test:shard:3": "pnpm exec playwright test --workers=4 --shard=3/3",
"ci:test:mysql": "E2E_DB_TYPE=mysql pnpm exec playwright test --workers=2",
"ci:test:pg": "E2E_DB_TYPE=pg pnpm exec playwright test --workers=2",
"preinstall": "npx only-allow pnpm"

Write Preview
Loading…
Cancel
Save