CI/CD improvements and archive files on shared system (#6469)

* archive files on shared system and remove parallel build, 3 shards

* add commented lines back in the file as it will be needed

.github/workflows/ci-cd.yml

@@ -118,19 +118,12 @@ jobs:
         db: mysql
         shard: 2
     playwright-mysql-3:
-      needs: playwright-mysql-1
+      needs: pre-build-for-playwright
       if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft )}}
       uses: ./.github/workflows/playwright-test-workflow.yml
       with:
         db: mysql
         shard: 3
-    playwright-mysql-4:
-      needs: playwright-mysql-2
-      if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft )}}
-      uses: ./.github/workflows/playwright-test-workflow.yml
-      with:
-        db: mysql
-        shard: 4                
     playwright-sqlite-1:
       needs: pre-build-for-playwright
       if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }}
@@ -145,6 +138,13 @@ jobs:
       with:
         db: sqlite
         shard: 2
+    playwright-sqlite-3:
+      needs: pre-build-for-playwright
+      if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }}
+      uses: ./.github/workflows/playwright-test-workflow.yml
+      with:
+        db: sqlite
+        shard: 3        
     playwright-pg-shard-1:
       needs: pre-build-for-playwright
       if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }}
@@ -160,16 +160,9 @@ jobs:
         db: pg
         shard: 2
     playwright-pg-shard-3:
-      needs: playwright-pg-shard-1
+      needs: pre-build-for-playwright
       if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft )}}
       uses: ./.github/workflows/playwright-test-workflow.yml
       with:
         db: pg
-        shard: 3    
+        shard: 3    
-    playwright-pg-shard-4:
-      needs: playwright-pg-shard-2
-      if: ${{ always() && ( github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft ) }}
-      uses: ./.github/workflows/playwright-test-workflow.yml
-      with:
-        db: pg
-        shard: 4                

.github/workflows/playwright-test-workflow.yml

@@ -13,37 +13,16 @@ on:
 
 jobs:
   playwright:
-    runs-on: [self-hosted, v2]
+    runs-on: [self-hosted, v3]
     timeout-minutes: 100
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v3       
-      - name: pre-requisite checks
-        shell: bash
-        run: |
-          node_version=$(node --version || echo "error")
-          pnpm_version=$(pnpm -v || echo "error")            
-          echo "node version: $node_version"
-          echo "pnpm version: $pnpm_version"
-          if [[ $node_version != v18* ]] || [[ $pnp_version != 8* ]]; then
-            echo "version mismatch: expected node v18 and pnpm v8"
-            RUN_PRERQUISITE_STEPS="true"  
-          elif [[ ${FORCE_RUN_PRERQUISITE_STEPS} == "true" ]];
-              # || [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
-          then
-            echo "FORCE_RUN_PRERQUISITE_STEPS is true"
-            RUN_PRERQUISITE_STEPS="true"
-          else
-            RUN_PRERQUISITE_STEPS="false"  
-          fi
-          echo "RUN_PRERQUISITE_STEPS=${RUN_PRERQUISITE_STEPS}" >> $GITHUB_ENV        
       - name: Setup Node
-        if: env.RUN_PRERQUISITE_STEPS == 'true'
         uses: actions/setup-node@v3
         with:
           node-version: 18.14.0
       - name: Setup pnpm
-        if: env.RUN_PRERQUISITE_STEPS == 'true'
         uses: pnpm/action-setup@v2
         with:
           version: 8
@@ -51,8 +30,21 @@ jobs:
         shell: bash
         run: |
           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+      - name: check if npm cache is needed
+        shell: bash
+        run: |        
+          IS_NPM_CACHE_DOWNLOAD_REQUIRED="/cache-marker-v1.txt"          
+          # update the above file name to force the cache ex: /cache-marker-v2.txt.
+          if [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
+          then
+            echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED is true"
+            IS_NPM_CACHE_DOWNLOAD_REQUIRED="true"
+          else
+            IS_NPM_CACHE_DOWNLOAD_REQUIRED="false"  
+          fi
+          echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED=${IS_NPM_CACHE_DOWNLOAD_REQUIRED}" >> $GITHUB_ENV           
       - uses: actions/cache@v3
-        if: env.RUN_PRERQUISITE_STEPS == 'true'
+        if: env.IS_NPM_CACHE_DOWNLOAD_REQUIRED == 'true'
         name: Setup pnpm cache
         with:
           path: ${{ env.STORE_PATH }}
@@ -113,22 +105,28 @@ jobs:
         working-directory: ./packages/nocodb
         run: |
           pnpm run watch:run:playwright:pg &> ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
-      - name: copy built output and start frontend
+      - name: download ui build from artf server
-        if: always()
+        working-directory: ./packages/nc-gui/
-        working-directory: ./
         run: |
           # expects the variables to be available in runner context.
-          path="gh-artifacts/runs/${GITHUB_RUN_ID}/ui-build/.output"
+          rm -rf .output.zip .output
-          target_dir="/mnt/${path}"
+          FILE="$(echo ${GITHUB_REPOSITORY} | sed "s,/,-,g")-${GITHUB_RUN_ID}.zip"
-          mkdir -p ${target_dir}
+          BUILD_FILE_URL=http://65.21.27.147/gh-artifacts/ui-builds/${FILE}
-          if [[ -d ${target_dir} ]]; then
+          time curl -o output.zip ${BUILD_FILE_URL} -n
-            echo "Directory ${target_dir} exists."
+          unzip -oq output.zip 
-            cp -r ${target_dir} ./packages/nc-gui/ || echo "playwright reports directory does not exists" >>  ${target_dir}/playwright-report/index.html
+
-          else
+          # path="gh-artifacts/runs/${GITHUB_RUN_ID}/ui-build/.output"
-            echo "Error: Directory ${target_dir} does not exists."
+          # target_dir="/mnt/${path}"
-            exit 1
+          # if [[ -d ${target_dir} ]]; then
-          fi
+          #   echo "Directory ${target_dir} exists."
-          cd ./packages/nc-gui/
+          #   cp -r ${target_dir} . || echo "playwright reports directory does not exists" >>  ${target_dir}/playwright-report/index.html
+          # else
+          #   echo "Error: Directory ${target_dir} does not exists."
+          #   exit 1
+          # fi
+      - name: start frontend
+        working-directory: ./packages/nc-gui/ 
+        run: |
           pnpm run ci:start
       - name: Install Playwright Browsers
         working-directory: ./tests/playwright
@@ -183,7 +181,8 @@ jobs:
 #      - name: Run quick tests
 #        if: ${{ inputs.db == 'sqlite' }}
 #        working-directory: ./tests/playwright
-#        run: PLAYWRIGHT_HTML_REPORT=playwright-report-quick npm run test:quick
+#        run: PLAYWRIGHT_HTML_REPORT=playwright-report-quick npm run test:quick        
+
       - uses: actions/upload-artifact@v3
         if: ${{ inputs.db == 'sqlite' }}
         with:
@@ -234,6 +233,6 @@ jobs:
           cp -r ./tests/playwright/playwright-report ${target_dir}/  || echo "playwright reports directory does not exists" >>  ${target_dir}/playwright-report/index.html
           cp ./packages/nocodb/*_test_backend.log ${target_dir}/ || echo "backend logs file does not exists" >> ${target_dir}/index.html
           # end: artifacts copy
-          SUMMARY='[Artifacts](http://135.181.48.96/'${path}')
+          SUMMARY='[Artifacts]('${REPORTS_HOST}/${path}')
-          [playwright-report](http://135.181.48.96/'${path}'/playwright-report)'
+          [playwright-report]('${REPORTS_HOST}/${path}'/playwright-report)'
           echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY            

.github/workflows/pre-build-for-playwright.yml

@@ -11,46 +11,38 @@ on:
 
 jobs:
   playwright:
-    runs-on: [self-hosted, aws]
+    runs-on: [self-hosted, v3]
     timeout-minutes: 100
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: pre-requisite checks
-        shell: bash
-        run: |
-          node_version=$(node --version || echo "error")
-          pnpm_version=$(pnpm -v || echo "error")          
-          echo "node version: $node_version"
-          echo "pnpm version: $pnpm_version"
-          if [[ $node_version != v18* ]] || [[ $pnp_version != 8* ]]; then
-            echo "version mismatch: expected node v18 and pnpm v8"
-            RUN_PRERQUISITE_STEPS="true"  
-          elif [[ ${FORCE_RUN_PRERQUISITE_STEPS} == "true" ]];
-              # || [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
-          then
-            echo "FORCE_RUN_PRERQUISITE_STEPS is true"
-            RUN_PRERQUISITE_STEPS="true"
-          else
-            RUN_PRERQUISITE_STEPS="false"  
-          fi
-          echo "RUN_PRERQUISITE_STEPS=${RUN_PRERQUISITE_STEPS}" >> $GITHUB_ENV
       - name: Setup Node
-        if: env.RUN_PRERQUISITE_STEPS == 'true'
         uses: actions/setup-node@v3
         with:
           node-version: 18.14.0
       - name: Setup pnpm
-        if: env.RUN_PRERQUISITE_STEPS == 'true'
         uses: pnpm/action-setup@v2
         with:
-          version: 8
+          version: 8        
+      - name: check if npm cache is needed
+        shell: bash
+        run: |        
+          PRE_REQ_CHECK_FILE_PATH="/cache-marker-v1.txt"          
+          # update the above file name to force the cache ex: /cache-marker-v2.txt.
+          if [[ ! -f ${PRE_REQ_CHECK_FILE_PATH} ]];
+          then
+            echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED is true"
+            IS_NPM_CACHE_DOWNLOAD_REQUIRED="true"
+          else
+            IS_NPM_CACHE_DOWNLOAD_REQUIRED="false"  
+          fi
+          echo "IS_NPM_CACHE_DOWNLOAD_REQUIRED=${IS_NPM_CACHE_DOWNLOAD_REQUIRED}" >> $GITHUB_ENV           
       - name: Get pnpm store directory
         shell: bash
         run: |
-          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+          echo "STORE_PATH=/root/setup-pnpm/node_modules/.bin/store/v3" >> $GITHUB_ENV          
       - uses: actions/cache@v3
-        if: env.RUN_PRERQUISITE_STEPS == 'true'
+        if: env.IS_NPM_CACHE_DOWNLOAD_REQUIRED == 'true'
         name: Setup pnpm cache
         with:
           path: ${{ env.STORE_PATH }}
@@ -59,22 +51,18 @@ jobs:
             ${{ runner.os }}-pnpm-store-
       - name: install dependencies
         run: pnpm bootstrap
-      - name: run frontend
+      - name: build frontend (nc-gui)
         working-directory: ./packages/nc-gui
         run: |
           pnpm run build
         timeout-minutes: 20
-      - name: Copy Artifacts to Local Artifacts Dir
+      - name: upload frontend (nc-gui) build to artf server
-        if: always()
         working-directory: ./
         run: |
           # expects the variables to be available in runner context.
-          path="gh-artifacts/runs/${GITHUB_RUN_ID}/ui-build/"
+          FILE="$(echo ${GITHUB_REPOSITORY} | sed "s,/,-,g")-${GITHUB_RUN_ID}.zip"
-          target_dir="/mnt/${path}"
+          cd ./packages/nc-gui/
-          mkdir -p ${target_dir}
+          zip -r ${FILE} .output || echo "UI build directory does not exists" >&2
-
+          echo "uploading  ${FILE} to http://65.21.27.147/upload/${FILE}"
-          # start : add any artifacts to be copied here
+          time curl -T "${FILE}" http://65.21.27.147/upload/${FILE} -n
-          cp -r ./packages/nc-gui/.output ${target_dir}/  || echo "playwright reports directory does not exists" >>  ${target_dir}/playwright-report/index.html
+          
-
-          SUMMARY='[UI BUILD](http://65.21.27.147/'${path}')'
-          echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY

tests/playwright/package.json

@@ -23,10 +23,6 @@
   "scripts": {
     "test": "TRACE=true pnpm exec playwright test --workers=4",
     "test:fast": "pnpm exec playwright test --workers=6",
-    "test:shard:1": "TRACE=true pnpm exec playwright test --workers=4 --shard=1/4",
-    "test:shard:2": "TRACE=true pnpm exec playwright test --workers=4 --shard=2/4",
-    "test:shard:3": "TRACE=true pnpm exec playwright test --workers=4 --shard=3/4",
-    "test:shard:4": "TRACE=true pnpm exec playwright test --workers=4 --shard=4/4",
     "test:repeat": "TRACE=true pnpm exec playwright test --workers=4 --repeat-each=12",
     "test:quick": "TRACE=true PW_QUICK_TEST=1 pnpm exec playwright test --workers=4",
     "test:debug": "./startPlayWrightServer.sh; PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 0 --workers 1 --max-failures=1",
@@ -34,10 +30,9 @@
     "test:debug:watch": "pnpm dlx nodemon -e ts -w ./ -x \"pnpm run test:debug\"",
     "test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1",
     "ci:test": "pnpm exec playwright test --workers=2",
-    "ci:test:shard:1": "pnpm exec playwright test --workers=4 --shard=1/4",
+    "ci:test:shard:1": "pnpm exec playwright test --workers=4 --shard=1/3",
-    "ci:test:shard:2": "pnpm exec playwright test --workers=4 --shard=2/4",
+    "ci:test:shard:2": "pnpm exec playwright test --workers=4 --shard=2/3",
-    "ci:test:shard:3": "pnpm exec playwright test --workers=4 --shard=3/4",
+    "ci:test:shard:3": "pnpm exec playwright test --workers=4 --shard=3/3",
-    "ci:test:shard:4": "pnpm exec playwright test --workers=4 --shard=4/4",
     "ci:test:mysql": "E2E_DB_TYPE=mysql pnpm exec playwright test --workers=2",
     "ci:test:pg": "E2E_DB_TYPE=pg pnpm exec playwright test --workers=2",
     "preinstall": "npx only-allow pnpm"