Merge pull request #4521 from nocodb/test/add-stress-testing-to-ci

Added stress testing of newly added test to CI

.github/workflows/ci-cd.yml

@@ -10,6 +10,7 @@ on:
             - "packages/nc-gui/**"
             - "packages/nocodb/**"
             - ".github/workflows/ci-cd.yml"
+            - "tests/playwright/**"
     pull_request:
         types: [opened, reopened, synchronize, ready_for_review, labeled]
         branches: [develop]

.github/workflows/playwright-test-workflow.yml

@@ -98,6 +98,14 @@ jobs:
         working-directory: ./tests/playwright
         run: E2E_DB_TYPE=${{ inputs.db }} npm run ci:test:shard:${{ inputs.shard }}
 
+      # Stress test added/modified tests
+      - name: Fetch develop branch
+        working-directory: ./tests/playwright
+        run: git fetch origin develop
+      - name: Stress test
+        working-directory: ./tests/playwright
+        run: E2E_DB_TYPE=${{ inputs.db }} node ./scripts/stressTestNewlyAddedTest.js
+
       # Quick tests (pg on sqlite shard 0 and sqlite on sqlite shard 1)
       - name: Run quick server and tests (pg)
         if: ${{ inputs.db == 'sqlite' && inputs.shard == '1' }}
@@ -132,13 +140,19 @@ jobs:
           name: playwright-report-quick-${{ inputs.shard }}
           path: ./tests/playwright/playwright-report-quick/
           retention-days: 2
-          
+
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-report-${{ inputs.db }}-${{ inputs.shard }}
           path: ./tests/playwright/playwright-report/
           retention-days: 2
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report-stress-${{ inputs.db }}-${{ inputs.shard }}
+          path: ./tests/playwright/playwright-report-stress/
+          retention-days: 2
       - uses: actions/upload-artifact@v3
         if: always()
         with:

tests/playwright/.gitignore

@@ -3,6 +3,7 @@ node_modules/
 /playwright-report/
 /playwright-report copy/
 /playwright-report-quick/
+/playwright-report-stress/
 /playwright/.cache/
 .env
 output

tests/playwright/scripts/stressTestNewlyAddedTest.js

@@ -0,0 +1,34 @@
+// eslint-disable-next-line no-undef
+const util = require('util');
+// eslint-disable-next-line no-undef
+const exec = util.promisify(require('child_process').exec);
+// Get items from `git diff develop'
+
+void (async () => {
+  const { stdout: allFileNames } = await exec('git diff --name-only origin/develop');
+  // return if no changed file ends with .js
+  const testFilesInChangedFiles = allFileNames.split('\n').filter(fileName => fileName.endsWith('.spec.ts'));
+  if (testFilesInChangedFiles.length === 0) {
+    console.log('No test file changed, skipping stress test');
+    return;
+  }
+
+  const { stdout } = await exec(`git diff origin/develop -- *.spec.ts **/*.spec.ts | grep test\\(`);
+  // eslint-disable-next-line no-undef
+  const dbType = process.env.E2E_DB_TYPE;
+
+  // get test names which is in the form of `+  test('test name', () => {'
+  const testNames = stdout
+    .match(/\+ {2}test\('(.*)',/g)
+    // extract test name by removing `+  test('` and `',*`
+    .map(testName => testName.replace("test('", '').trimEnd().slice(0, -2).slice(1, testName.length).trim());
+  console.log({ dbType, testNames });
+
+  // run all the tests by title using regex with exact match
+  const { stdout: pwStdout } = await exec(
+    `PLAYWRIGHT_HTML_REPORT=playwright-report-stress E2E_DB_TYPE=${dbType} npx playwright test --repeat-each=2 --workers=2 -g "${testNames.join(
+      '|'
+    )}"`
+  );
+  console.log('pwStdout:', pwStdout);
+})();