Merge pull request #6230 from nocodb/pw_tests_sh

pw tests on sh runner

.github/workflows/playwright-test-workflow.yml

@@ -13,21 +13,9 @@ on:
 
 jobs:
   playwright:
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, v2]
-    timeout-minutes: 40
+    timeout-minutes: 100
     steps:
-      # Reference: https://github.com/pierotofy/set-swap-space/blob/master/action.yml
-      - name: Set 5gb swap
-        shell: bash
-        # Delete the swap file, allocate a new one, and activate it
-        run: |
-            export SWAP_FILE=$(swapon --show=NAME | tail -n 1)
-            sudo swapoff $SWAP_FILE
-            sudo rm $SWAP_FILE
-            sudo fallocate -l 5G $SWAP_FILE
-            sudo chmod 600 $SWAP_FILE
-            sudo mkswap $SWAP_FILE
-            sudo swapon $SWAP_FILE
       - name: Setup Node
         uses: actions/setup-node@v3
         with:
@@ -42,11 +30,31 @@ jobs:
         with:
           # npm cache files are stored in `~/.npm` on Linux/macOS
           path: ~/.npm
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          key: ${{ runner.os }}-v2-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-v2-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
+            ${{ runner.os }}-v2-build-
-            ${{ runner.os }}-
+            ${{ runner.os }}-v2
+      - name: setup pg
+        if: ${{ inputs.db == 'pg' || ( inputs.db == 'sqlite' && inputs.shard == '1' ) }}
+        working-directory: ./
+        run:  |
+          service postgresql start
+          cd /var/lib/postgresql/ && sudo -u postgres psql -c "SELECT 'dropdb '||datname||'' FROM pg_database WHERE datistemplate = false AND datallowconn = true And datname NOT IN ('postgres')" |grep ' dropdb ' |  sudo -u postgres /bin/bash ; cd
+          sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD 'password';" 
+          sudo -u postgres psql -c "ALTER USER postgres WITH SUPERUSER;" 
+          service postgresql restart
+      - name: Set CI env
+        run: export CI=true
+      - name: Kill stale servers
+        run: |
+          # export NODE_OPTIONS=\"--max_old_space_size=16384\";
+          kill -9 $(lsof -t -i:8080) || echo "no process running on 8080"
+          kill -9 $(lsof -t -i:3000) || echo "no process running on 3000"
+      - name: Set CI env
+        run: export CI=true
+      - name: Set NC Edition
+        run: export EE=true
       - name: install dependencies nocodb-sdk
         working-directory: ./packages/nocodb-sdk
         run: npm install
@@ -55,16 +63,23 @@ jobs:
         run: npm run build
       - name: Setup mysql
         if: ${{ inputs.db == 'mysql' }}
-        working-directory: ./
+        working-directory: ./packages/nocodb/tests/mysql-sakila-db
-        run: docker-compose -f ./tests/playwright/scripts/docker-compose-mysql-playwright.yml up -d &
+        run: |
-      - name: setup pg
+          # Get a list of non-system databases and construct the DROP DATABASE statement for each
-        if: ${{ inputs.db == 'pg' }}
+          service mysql start
-        working-directory: ./
+          mysql -u'root' -p'password' -e "SHOW DATABASES" --skip-column-names | grep -Ev "(information_schema|mysql|performance_schema|sys)" | while read db; do
-        run: docker-compose -f ./tests/playwright/scripts/docker-compose-playwright-pg.yml up -d &
+          mysql -u'root' -p'password' -e "DROP DATABASE IF EXISTS \`$db\`";
-      - name: setup pg for quick tests
+          done
+          # keep sql_mode default except remove "STRICT_TRANS_TABLES"
+          mysql -u'root' -p'password' -e "SET GLOBAL sql_mode = 'ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';"
+          # this is only needed for connecting to sakila db as its refeferred in multiple places in test code          
+          mysql -u'root' -p'password' < 01-mysql-sakila-schema.sql   
+          mysql -u'root' -p'password' < 02-mysql-sakila-insert-data.sql
+      - name: Setup pg for quick tests
         if: ${{ inputs.db == 'sqlite' && inputs.shard == '1' }}
-        working-directory: ./
+        working-directory: ./packages/nocodb/tests/pg-cy-quick/
-        run: docker-compose -f ./tests/playwright/scripts/docker-compose-pg-pw-quick.yml up -d &
+        run: |
+          sudo -u postgres psql -U postgres -f 01-cy-quick.sql
       - name: run frontend
         working-directory: ./packages/nc-gui
         run: npm run ci:run
@@ -74,26 +89,28 @@ jobs:
         working-directory: ./packages/nocodb
         run: |
           npm install
-          npm run watch:run:playwright > ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
+          npm run watch:run:playwright &> ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
       - name: Run backend:mysql
         if: ${{ inputs.db == 'mysql' }}
         working-directory: ./packages/nocodb
         run: |
           npm install
-          npm run watch:run:playwright:mysql > ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
+          npm run watch:run:playwright:mysql &> ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
       - name: Run backend:pg
         if: ${{ inputs.db == 'pg' }}
         working-directory: ./packages/nocodb
         run: |
           npm install
-          npm run watch:run:playwright:pg > ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
+          npm run watch:run:playwright:pg &> ${{ inputs.db }}_${{ inputs.shard }}_test_backend.log &
       - name: Cache playwright npm modules
         uses: actions/cache@v3
         id: playwright-cache
         with:
           path: |
             **/tests/playwright/node_modules
-          key: cache-nc-playwright-${{ hashFiles('**/tests/playwright/package-lock.json') }}
+          key: cache-v2-nc-playwright-${{ hashFiles('**/tests/playwright/package-lock.json') }}
+          restore-keys: |
+            cache-v2-nc-playwright-
       - name: Install dependencies
         if: steps.playwright-cache.outputs.cache-hit != 'true'
         working-directory: ./tests/playwright
@@ -120,20 +137,20 @@ jobs:
         working-directory: ./tests/playwright
         run: E2E_DB_TYPE=${{ inputs.db }} node ./scripts/stressTestNewlyAddedTest.js
 
-      # Quick tests (pg on sqlite shard 0 and sqlite on sqlite shard 1)
+#      # Quick tests (pg on sqlite shard 0 and sqlite on sqlite shard 1)
 #      - name: Run quick server and tests (pg)
 #        if: ${{ inputs.db == 'sqlite' && inputs.shard == '1' }}
 #        working-directory: ./packages/nocodb
 #        run: |
 #          kill -9 $(lsof -t -i:8080)
-#          npm run watch:run:playwright:pg:cyquick &
+#          npm run watch:run:playwright:pg:cyquick > quick_${{ inputs.shard }}_test_backend.log &
 #      - name: Run quick server and tests (sqlite)
 #        if: ${{ inputs.db == 'sqlite' && inputs.shard == '2' }}
 #        working-directory: ./packages/nocodb
 #        run: |
 #          kill -9 $(lsof -t -i:8080)
 #          npm run watch:run:playwright:quick > quick_${{ inputs.shard }}_test_backend.log &
-#      - name: Wait for backend & run quick tests
+#      - name: Wait for backend for sqlite-tests
 #        if: ${{ inputs.db == 'sqlite' }}
 #        working-directory: ./tests/playwright
 #        run: |
@@ -141,7 +158,11 @@ jobs:
 #            printf '.'
 #            sleep 2
 #          done
-#          PLAYWRIGHT_HTML_REPORT=playwright-report-quick npm run test:quick
+#        timeout-minutes: 1
+#      - name: Run quick tests
+#        if: ${{ inputs.db == 'sqlite' }}
+#        working-directory: ./tests/playwright
+#        run: PLAYWRIGHT_HTML_REPORT=playwright-report-quick npm run test:quick
       - uses: actions/upload-artifact@v3
         if: ${{ inputs.db == 'sqlite' }}
         with:
@@ -173,3 +194,9 @@ jobs:
           name: backend-logs-${{ inputs.db }}-${{ inputs.shard }}
           path: ./packages/nocodb/${{ inputs.db }}_${{ inputs.shard }}_test_backend.log
           retention-days: 2
+      - name: stop database servers
+        if: always()
+        working-directory: ./packages/nocodb
+        run: |
+          service postgresql stop
+          service mysql stop