From f8c077bdbf66a1c52bc67c4191fefd0e61b1cf45 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Fri, 29 Sep 2023 23:29:55 +0530 Subject: [PATCH 01/11] fix: Reduced pw worker to test sqlite test stability --- .github/workflows/ci-cd.yml | 112 +++++++++++++++++----------------- tests/playwright/package.json | 8 +-- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index fcf267fdef..8d4e2b637d 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -103,34 +103,34 @@ jobs: pre-build-for-playwright: uses: ./.github/workflows/pre-build-for-playwright.yml - playwright-mysql-1: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: mysql - shard: 1 - playwright-mysql-2: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: mysql - shard: 2 - playwright-mysql-3: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: mysql - shard: 3 - playwright-mysql-4: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: mysql - shard: 4 + # playwright-mysql-1: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: mysql + # shard: 1 + # playwright-mysql-2: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: mysql + # shard: 2 + # playwright-mysql-3: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: mysql + # shard: 3 + # playwright-mysql-4: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: mysql + # shard: 4 playwright-sqlite-1: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -159,31 +159,31 @@ jobs: with: db: sqlite shard: 4 - playwright-pg-shard-1: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: pg - shard: 1 - playwright-pg-shard-2: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: pg - shard: 2 - playwright-pg-shard-3: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: pg - shard: 3 - playwright-pg-shard-4: - needs: pre-build-for-playwright - if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - uses: ./.github/workflows/playwright-test-workflow.yml - with: - db: pg - shard: 4 \ No newline at end of file + # playwright-pg-shard-1: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: pg + # shard: 1 + # playwright-pg-shard-2: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: pg + # shard: 2 + # playwright-pg-shard-3: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: pg + # shard: 3 + # playwright-pg-shard-4: + # needs: pre-build-for-playwright + # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + # uses: ./.github/workflows/playwright-test-workflow.yml + # with: + # db: pg + # shard: 4 \ No newline at end of file diff --git a/tests/playwright/package.json b/tests/playwright/package.json index 3d965cc05e..cae96ae78c 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -30,10 +30,10 @@ "test:debug:watch": "pnpm dlx nodemon -e ts -w ./ -x \"pnpm run test:debug\"", "test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1", "ci:test": "pnpm exec playwright test --workers=2", - "ci:test:shard:1": "pnpm exec playwright test --workers=2 --shard=1/4", - "ci:test:shard:2": "pnpm exec playwright test --workers=2 --shard=2/4", - "ci:test:shard:3": "pnpm exec playwright test --workers=2 --shard=3/4", - "ci:test:shard:4": "pnpm exec playwright test --workers=2 --shard=4/4", + "ci:test:shard:1": "pnpm exec playwright test --workers=1 --shard=1/4", + "ci:test:shard:2": "pnpm exec playwright test --workers=1 --shard=2/4", + "ci:test:shard:3": "pnpm exec playwright test --workers=1 --shard=3/4", + "ci:test:shard:4": "pnpm exec playwright test --workers=1 --shard=4/4", "ci:test:flaky:repeat": "pnpm exec playwright test --workers=4 --grep @flaky --repeat-each=3", "ci:test:mysql": "E2E_DB_TYPE=mysql pnpm exec playwright test --workers=2", "ci:test:pg": "E2E_DB_TYPE=pg pnpm exec playwright test --workers=2", From e2dbd795f0dcd54e39c5922f2acb017d1eba45a4 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 00:10:08 +0530 Subject: [PATCH 02/11] fix: Reduced pw worker to test sqlite test stability --- .github/workflows/ci-cd.yml | 112 ++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 8d4e2b637d..b7bfb0eb53 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -103,34 +103,34 @@ jobs: pre-build-for-playwright: uses: ./.github/workflows/pre-build-for-playwright.yml - # playwright-mysql-1: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: mysql - # shard: 1 - # playwright-mysql-2: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: mysql - # shard: 2 - # playwright-mysql-3: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: mysql - # shard: 3 - # playwright-mysql-4: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: mysql - # shard: 4 + playwright-mysql-1: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 1 + playwright-mysql-2: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 2 + playwright-mysql-3: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 3 + playwright-mysql-4: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 4 playwright-sqlite-1: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -159,31 +159,31 @@ jobs: with: db: sqlite shard: 4 - # playwright-pg-shard-1: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: pg - # shard: 1 - # playwright-pg-shard-2: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: pg - # shard: 2 - # playwright-pg-shard-3: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: pg - # shard: 3 - # playwright-pg-shard-4: - # needs: pre-build-for-playwright - # if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} - # uses: ./.github/workflows/playwright-test-workflow.yml - # with: - # db: pg - # shard: 4 \ No newline at end of file + playwright-pg-shard-1: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 1 + playwright-pg-shard-2: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 2 + playwright-pg-shard-3: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 3 + playwright-pg-shard-4: + needs: pre-build-for-playwright + if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} + uses: ./.github/workflows/playwright-test-workflow.yml + with: + db: sqlite + shard: 4 \ No newline at end of file From 1fb5d128c0423ed369f6eb292d648eb9df7673c1 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 09:31:29 +0530 Subject: [PATCH 03/11] fix: Increased worker count and increased test action timeout to test stability --- tests/playwright/package.json | 8 ++++---- tests/playwright/playwright.config.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/playwright/package.json b/tests/playwright/package.json index cae96ae78c..f6e64b0815 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -30,10 +30,10 @@ "test:debug:watch": "pnpm dlx nodemon -e ts -w ./ -x \"pnpm run test:debug\"", "test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1", "ci:test": "pnpm exec playwright test --workers=2", - "ci:test:shard:1": "pnpm exec playwright test --workers=1 --shard=1/4", - "ci:test:shard:2": "pnpm exec playwright test --workers=1 --shard=2/4", - "ci:test:shard:3": "pnpm exec playwright test --workers=1 --shard=3/4", - "ci:test:shard:4": "pnpm exec playwright test --workers=1 --shard=4/4", + "ci:test:shard:1": "pnpm exec playwright test --workers=3 --shard=1/4", + "ci:test:shard:2": "pnpm exec playwright test --workers=3 --shard=2/4", + "ci:test:shard:3": "pnpm exec playwright test --workers=3 --shard=3/4", + "ci:test:shard:4": "pnpm exec playwright test --workers=3 --shard=4/4", "ci:test:flaky:repeat": "pnpm exec playwright test --workers=4 --grep @flaky --repeat-each=3", "ci:test:mysql": "E2E_DB_TYPE=mysql pnpm exec playwright test --workers=2", "ci:test:pg": "E2E_DB_TYPE=pg pnpm exec playwright test --workers=2", diff --git a/tests/playwright/playwright.config.ts b/tests/playwright/playwright.config.ts index 8f7359dca8..5ea4151fd1 100644 --- a/tests/playwright/playwright.config.ts +++ b/tests/playwright/playwright.config.ts @@ -38,7 +38,7 @@ export default defineConfig({ // headless: !!process.env.CI, headless: true, /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ - actionTimeout: 4000, + actionTimeout: process.env.E2E_DB_TYPE === 'sqlite' ? 12000 : 4000, /* Base URL to use in actions like `await page.goto('/')`. */ baseURL: 'http://localhost:3000', From 28906f1ac13929c3054fe553dd4f3fafc948a8f6 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 10:10:57 +0530 Subject: [PATCH 04/11] fix: Increased worker count and increased test action timeout to test stability --- tests/playwright/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/playwright/package.json b/tests/playwright/package.json index f6e64b0815..3d965cc05e 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -30,10 +30,10 @@ "test:debug:watch": "pnpm dlx nodemon -e ts -w ./ -x \"pnpm run test:debug\"", "test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1", "ci:test": "pnpm exec playwright test --workers=2", - "ci:test:shard:1": "pnpm exec playwright test --workers=3 --shard=1/4", - "ci:test:shard:2": "pnpm exec playwright test --workers=3 --shard=2/4", - "ci:test:shard:3": "pnpm exec playwright test --workers=3 --shard=3/4", - "ci:test:shard:4": "pnpm exec playwright test --workers=3 --shard=4/4", + "ci:test:shard:1": "pnpm exec playwright test --workers=2 --shard=1/4", + "ci:test:shard:2": "pnpm exec playwright test --workers=2 --shard=2/4", + "ci:test:shard:3": "pnpm exec playwright test --workers=2 --shard=3/4", + "ci:test:shard:4": "pnpm exec playwright test --workers=2 --shard=4/4", "ci:test:flaky:repeat": "pnpm exec playwright test --workers=4 --grep @flaky --repeat-each=3", "ci:test:mysql": "E2E_DB_TYPE=mysql pnpm exec playwright test --workers=2", "ci:test:pg": "E2E_DB_TYPE=pg pnpm exec playwright test --workers=2", From e8a07675cd04e7b6c37cc6844bc7c44654c28de5 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 10:42:08 +0530 Subject: [PATCH 05/11] fix: Increased worker count and increased test action timeout to test stability --- .github/workflows/ci-cd.yml | 38 ++++++++++++------- .../workflows/playwright-test-workflow.yml | 5 ++- tests/playwright/package.json | 8 ++-- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b7bfb0eb53..babd895ead 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -108,29 +108,33 @@ jobs: if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite + db: mysql shard: 1 + workers: 4 playwright-mysql-2: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite + db: mysql shard: 2 + workers: 4 playwright-mysql-3: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite + db: mysql shard: 3 + workers: 4 playwright-mysql-4: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite - shard: 4 + db: mysql + shard: 4 + workers: 4 playwright-sqlite-1: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -138,6 +142,7 @@ jobs: with: db: sqlite shard: 1 + workers: 1 playwright-sqlite-2: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -145,45 +150,52 @@ jobs: with: db: sqlite shard: 2 + workers: 1 playwright-sqlite-3: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: db: sqlite - shard: 3 + shard: 3 + workers: 1 playwright-sqlite-4: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: db: sqlite - shard: 4 + shard: 4 + workers: 1 playwright-pg-shard-1: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite + db: pg shard: 1 + workers: 4 playwright-pg-shard-2: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite + db: pg shard: 2 + workers: 4 playwright-pg-shard-3: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite - shard: 3 + db: pg + shard: 3 + workers: 4 playwright-pg-shard-4: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: - db: sqlite - shard: 4 \ No newline at end of file + db: pg + shard: 4 + workers: 4 \ No newline at end of file diff --git a/.github/workflows/playwright-test-workflow.yml b/.github/workflows/playwright-test-workflow.yml index f753fec7f8..91db353fe5 100644 --- a/.github/workflows/playwright-test-workflow.yml +++ b/.github/workflows/playwright-test-workflow.yml @@ -10,6 +10,9 @@ on: db: required: true type: string + workers: + required: true + type: string jobs: playwright: @@ -144,7 +147,7 @@ jobs: timeout-minutes: 2 - name: Run Playwright Tests working-directory: ./tests/playwright - run: E2E_DB_TYPE=${{ inputs.db }} npm run ci:test:shard:${{ inputs.shard }} + run: E2E_DB_TYPE=${{ inputs.db }} npm run ci:test:shard:${{ inputs.shard }} --workers=${{ inputs.workers }} timeout-minutes: 60 # Stress test added/modified tests - name: Fetch develop branch diff --git a/tests/playwright/package.json b/tests/playwright/package.json index 3d965cc05e..3a2d733b79 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -30,10 +30,10 @@ "test:debug:watch": "pnpm dlx nodemon -e ts -w ./ -x \"pnpm run test:debug\"", "test:debug:quick:sqlite": "./startPlayWrightServer.sh; PW_QUICK_TEST=1 PW_TEST_REUSE_CONTEXT=1 PW_TEST_CONNECT_WS_ENDPOINT=ws://127.0.0.1:31000/ PWDEBUG=console pnpm exec playwright test -c playwright.config.ts --headed --project=chromium --retries 0 --timeout 5 --workers 1 --max-failures=1", "ci:test": "pnpm exec playwright test --workers=2", - "ci:test:shard:1": "pnpm exec playwright test --workers=2 --shard=1/4", - "ci:test:shard:2": "pnpm exec playwright test --workers=2 --shard=2/4", - "ci:test:shard:3": "pnpm exec playwright test --workers=2 --shard=3/4", - "ci:test:shard:4": "pnpm exec playwright test --workers=2 --shard=4/4", + "ci:test:shard:1": "pnpm exec playwright test --shard=1/4", + "ci:test:shard:2": "pnpm exec playwright test --shard=2/4", + "ci:test:shard:3": "pnpm exec playwright test --shard=3/4", + "ci:test:shard:4": "pnpm exec playwright test --shard=4/4", "ci:test:flaky:repeat": "pnpm exec playwright test --workers=4 --grep @flaky --repeat-each=3", "ci:test:mysql": "E2E_DB_TYPE=mysql pnpm exec playwright test --workers=2", "ci:test:pg": "E2E_DB_TYPE=pg pnpm exec playwright test --workers=2", From d6482fa6bf0dbaed0966c7a3984b94de6adbc8a9 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 11:23:20 +0530 Subject: [PATCH 06/11] fix: Increased worker count and increased test action timeout to test stability --- .github/workflows/ci-cd.yml | 16 ++-------------- .github/workflows/playwright-test-workflow.yml | 5 +---- tests/playwright/playwright.config.ts | 13 ++++++++++--- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index babd895ead..388cb7c8a6 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -110,7 +110,6 @@ jobs: with: db: mysql shard: 1 - workers: 4 playwright-mysql-2: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -118,7 +117,6 @@ jobs: with: db: mysql shard: 2 - workers: 4 playwright-mysql-3: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -126,7 +124,6 @@ jobs: with: db: mysql shard: 3 - workers: 4 playwright-mysql-4: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -134,7 +131,6 @@ jobs: with: db: mysql shard: 4 - workers: 4 playwright-sqlite-1: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -142,7 +138,6 @@ jobs: with: db: sqlite shard: 1 - workers: 1 playwright-sqlite-2: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -150,7 +145,6 @@ jobs: with: db: sqlite shard: 2 - workers: 1 playwright-sqlite-3: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -158,15 +152,13 @@ jobs: with: db: sqlite shard: 3 - workers: 1 playwright-sqlite-4: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: db: sqlite - shard: 4 - workers: 1 + shard: 4 playwright-pg-shard-1: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -174,7 +166,6 @@ jobs: with: db: pg shard: 1 - workers: 4 playwright-pg-shard-2: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -182,7 +173,6 @@ jobs: with: db: pg shard: 2 - workers: 4 playwright-pg-shard-3: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} @@ -190,12 +180,10 @@ jobs: with: db: pg shard: 3 - workers: 4 playwright-pg-shard-4: needs: pre-build-for-playwright if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'trigger-CI') || !github.event.pull_request.draft }} uses: ./.github/workflows/playwright-test-workflow.yml with: db: pg - shard: 4 - workers: 4 \ No newline at end of file + shard: 4 \ No newline at end of file diff --git a/.github/workflows/playwright-test-workflow.yml b/.github/workflows/playwright-test-workflow.yml index 91db353fe5..f753fec7f8 100644 --- a/.github/workflows/playwright-test-workflow.yml +++ b/.github/workflows/playwright-test-workflow.yml @@ -10,9 +10,6 @@ on: db: required: true type: string - workers: - required: true - type: string jobs: playwright: @@ -147,7 +144,7 @@ jobs: timeout-minutes: 2 - name: Run Playwright Tests working-directory: ./tests/playwright - run: E2E_DB_TYPE=${{ inputs.db }} npm run ci:test:shard:${{ inputs.shard }} --workers=${{ inputs.workers }} + run: E2E_DB_TYPE=${{ inputs.db }} npm run ci:test:shard:${{ inputs.shard }} timeout-minutes: 60 # Stress test added/modified tests - name: Fetch develop branch diff --git a/tests/playwright/playwright.config.ts b/tests/playwright/playwright.config.ts index 5ea4151fd1..e6ce067c3f 100644 --- a/tests/playwright/playwright.config.ts +++ b/tests/playwright/playwright.config.ts @@ -10,6 +10,14 @@ require('dotenv').config(); * See https://playwright.dev/docs/test-configuration. */ +let workers = process.env.CI ? 2 : 4; + +if (process.env.CI) { + if (process.env.E2E_DB_TYPE === 'sqlite') workers = 1; + if (process.env.E2E_DB_TYPE === 'mysql') workers = 4; + if (process.env.E2E_DB_TYPE === 'pg') workers = 4; +} + export default defineConfig({ testDir: process.env.PW_QUICK_TEST ? './quickTests' : './tests', /* Maximum time one test can run for. */ @@ -29,8 +37,7 @@ export default defineConfig({ /* Retry on CI only */ retries: process.env.CI ? 3 : 0, /* Opt out of parallel tests on CI. */ - // workers: process.env.CI ? 2 : 4, - workers: process.env.CI ? 2 : 4, + workers: workers, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ @@ -38,7 +45,7 @@ export default defineConfig({ // headless: !!process.env.CI, headless: true, /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ - actionTimeout: process.env.E2E_DB_TYPE === 'sqlite' ? 12000 : 4000, + actionTimeout: 4000, /* Base URL to use in actions like `await page.goto('/')`. */ baseURL: 'http://localhost:3000', From a2b29de8ea0930293fa83232cf8fbdb4e99f40ca Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 12:05:36 +0530 Subject: [PATCH 07/11] fix: Increased worker count and increased test action timeout to test stability --- tests/playwright/playwright.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/playwright/playwright.config.ts b/tests/playwright/playwright.config.ts index e6ce067c3f..019952c9bc 100644 --- a/tests/playwright/playwright.config.ts +++ b/tests/playwright/playwright.config.ts @@ -14,8 +14,8 @@ let workers = process.env.CI ? 2 : 4; if (process.env.CI) { if (process.env.E2E_DB_TYPE === 'sqlite') workers = 1; - if (process.env.E2E_DB_TYPE === 'mysql') workers = 4; - if (process.env.E2E_DB_TYPE === 'pg') workers = 4; + if (process.env.E2E_DB_TYPE === 'mysql') workers = 3; + if (process.env.E2E_DB_TYPE === 'pg') workers = 3; } export default defineConfig({ From ccc6f806d0b0ebde95596ef76f9b213b5680f935 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 12:45:21 +0530 Subject: [PATCH 08/11] fix: Cleanup --- tests/playwright/playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright/playwright.config.ts b/tests/playwright/playwright.config.ts index 019952c9bc..c7acdceb90 100644 --- a/tests/playwright/playwright.config.ts +++ b/tests/playwright/playwright.config.ts @@ -10,7 +10,7 @@ require('dotenv').config(); * See https://playwright.dev/docs/test-configuration. */ -let workers = process.env.CI ? 2 : 4; +let workers = 4; if (process.env.CI) { if (process.env.E2E_DB_TYPE === 'sqlite') workers = 1; From b85961030c722ab90fc21ad57483e8ffa620d4d6 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 13:12:24 +0530 Subject: [PATCH 09/11] fix: Fixed flakyness with row height test --- tests/playwright/pages/Dashboard/Grid/Row.ts | 4 ++-- tests/playwright/tests/db/features/undo-redo.spec.ts | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/playwright/pages/Dashboard/Grid/Row.ts b/tests/playwright/pages/Dashboard/Grid/Row.ts index 664d51260f..3f0ff8cbd5 100644 --- a/tests/playwright/pages/Dashboard/Grid/Row.ts +++ b/tests/playwright/pages/Dashboard/Grid/Row.ts @@ -13,13 +13,13 @@ export class RowPageObject extends BasePage { return this.rootPage.locator('tr.nc-grid-row'); } - async getRecord(index: number) { + getRecord(index: number) { return this.get().nth(index); } // style="height: 3rem;" async getRecordHeight(index: number) { - const record = await this.getRecord(index); + const record = this.getRecord(index); const style = await record.getAttribute('style'); return style.split(':')[1].split(';')[0].trim(); } diff --git a/tests/playwright/tests/db/features/undo-redo.spec.ts b/tests/playwright/tests/db/features/undo-redo.spec.ts index 20939be061..b4102665e2 100644 --- a/tests/playwright/tests/db/features/undo-redo.spec.ts +++ b/tests/playwright/tests/db/features/undo-redo.spec.ts @@ -294,9 +294,7 @@ test.describe('Undo Redo', () => { test('Row height', async ({ page }) => { async function verifyRowHeight({ height }: { height: string }) { - await dashboard.grid.rowPage.getRecordHeight(0).then(readValue => { - expect(readValue).toBe(height); - }); + await expect(dashboard.grid.rowPage.getRecord(0)).toHaveAttribute('style', `height: ${height};`); } // close 'Team & Auth' tab From e463481d6604c50cacfdc954ca0faa880cd9753d Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 13:57:25 +0530 Subject: [PATCH 10/11] fix: Fixed flakyness with form test --- tests/playwright/pages/Dashboard/Form/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/playwright/pages/Dashboard/Form/index.ts b/tests/playwright/pages/Dashboard/Form/index.ts index e84b0bc2b5..74335f7f6c 100644 --- a/tests/playwright/pages/Dashboard/Form/index.ts +++ b/tests/playwright/pages/Dashboard/Form/index.ts @@ -138,6 +138,8 @@ export class FormPage extends BasePage { } async removeAllFields() { + // TODO: Otherwise form input boxes are not visible sometimes + await this.rootPage.waitForTimeout(1000); await this.removeAllButton.click(); } From ee8411f370139323c26fe46a0f21f130c6e7a1c8 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Sat, 30 Sep 2023 13:59:03 +0530 Subject: [PATCH 11/11] fix: Fixed flakyness with form test --- tests/playwright/pages/Dashboard/Form/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/playwright/pages/Dashboard/Form/index.ts b/tests/playwright/pages/Dashboard/Form/index.ts index 74335f7f6c..1f2131377b 100644 --- a/tests/playwright/pages/Dashboard/Form/index.ts +++ b/tests/playwright/pages/Dashboard/Form/index.ts @@ -105,6 +105,9 @@ export class FormPage extends BasePage { } async reorderFields({ sourceField, destinationField }: { sourceField: string; destinationField: string }) { + // TODO: Otherwise form input boxes are not visible sometimes + await this.rootPage.waitForTimeout(650); + await expect(this.get().locator(`.nc-form-drag-${sourceField}`)).toBeVisible(); await expect(this.get().locator(`.nc-form-drag-${destinationField}`)).toBeVisible(); const src = this.get().locator(`.nc-form-drag-${sourceField.replace(' ', '')}`); @@ -113,6 +116,9 @@ export class FormPage extends BasePage { } async removeField({ field, mode }: { mode: string; field: string }) { + // TODO: Otherwise form input boxes are not visible sometimes + await this.rootPage.waitForTimeout(650); + if (mode === 'dragDrop') { const src = this.get().locator(`.nc-form-drag-${field.replace(' ', '')}`); const dst = this.get().locator(`[data-testid="nc-drag-n-drop-to-hide"]`); @@ -124,6 +130,9 @@ export class FormPage extends BasePage { } async addField({ field, mode }: { mode: string; field: string }) { + // TODO: Otherwise form input boxes are not visible sometimes + await this.rootPage.waitForTimeout(650); + if (mode === 'dragDrop') { const src = this.get().locator(`[data-testid="nc-form-hidden-column-${field}"] > div.ant-card-body`); const dst = this.get().locator(`[data-testid="nc-form-input-Country"]`); @@ -140,6 +149,7 @@ export class FormPage extends BasePage { async removeAllFields() { // TODO: Otherwise form input boxes are not visible sometimes await this.rootPage.waitForTimeout(1000); + await this.removeAllButton.click(); }