Retry E2E tests up to 3 times (#711)

.github/workflows/ci.yml

@@ -128,7 +128,6 @@ jobs:
           path: failed-*
   endtoend:
     name: "E2E"
-    needs: [cleanupaccounts]
     runs-on: ubuntu-latest
     env:
       NODE_TLS_REJECT_UNAUTHORIZED: 0
@@ -156,33 +155,13 @@ jobs:
       - name: ${{ matrix['test-file'] }}
         run: |
           # Run tests up to three times
-          n=0
+          for i in $(seq 1 3); do npx jest -c ./jest.config.playwright.js ${{ matrix['test-file'] }} && s=0 && break || s=$? && sleep 1; done; (exit $s)
-          until [ "$n" -ge 3 ]
-          do
-            npx jest -c ./jest.config.playwright.js ${{ matrix['test-file'] }}  && break
-            n=$((n+1)) 
-            sleep 1
-          done
         shell: bash
       - uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: screenshots
           path: screenshots/
-  cleanupaccounts:
-    name: "Cleanup Test Database Accounts"
-    runs-on: ubuntu-latest
-    env:
-      NOTEBOOKS_TEST_RUNNER_CLIENT_ID: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_ID }}
-      NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 14.x
-      - run: npm ci
-      - run: node utils/cleanupDBs.js
   nuget:
     name: Publish Nuget
     if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/') || contains(github.ref, 'release/')

.github/workflows/cleanup.yml

@@ -0,0 +1,28 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Cleanup End to End Account Resources
+
+on:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+  schedule:
+    # Once every hour
+    - cron: "0 * * * *"
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  cleanupaccounts:
+    name: "Cleanup Test Database Accounts"
+    runs-on: ubuntu-latest
+    env:
+      NOTEBOOKS_TEST_RUNNER_CLIENT_ID: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_ID }}
+      NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: npm ci
+      - run: node utils/cleanupDBs.js

test/mongo/container.spec.ts

@@ -1,11 +1,11 @@
 import { jest } from "@jest/globals";
 import "expect-playwright";
 import { safeClick } from "../utils/safeClick";
-import { generateUniqueName } from "../utils/shared";
+import { generateDatabaseNameWithTimestamp, generateUniqueName } from "../utils/shared";
 jest.setTimeout(240000);
 
-test("SQL CRUD", async () => {
+test("Mongo CRUD", async () => {
-  const databaseId = generateUniqueName("db");
+  const databaseId = generateDatabaseNameWithTimestamp();
   const containerId = generateUniqueName("container");
 
   await page.goto("https://localhost:1234/testExplorer.html?accountName=portal-mongo-runner");

test/sql/resourceToken.spec.ts

@@ -3,7 +3,7 @@ import { CosmosClient, PermissionMode } from "@azure/cosmos";
 import * as msRestNodeAuth from "@azure/ms-rest-nodeauth";
 import { jest } from "@jest/globals";
 import "expect-playwright";
-import { generateDatabaseName, generateUniqueName } from "../utils/shared";
+import { generateUniqueName } from "../utils/shared";
 jest.setTimeout(120000);
 
 const clientId = "fd8753b0-0707-4e32-84e9-2532af865fb4";
@@ -17,7 +17,7 @@ test("Resource token", async () => {
   const armClient = new CosmosDBManagementClient(credentials, subscriptionId);
   const account = await armClient.databaseAccounts.get(resourceGroupName, "portal-sql-runner");
   const keys = await armClient.databaseAccounts.listKeys(resourceGroupName, "portal-sql-runner");
-  const dbId = generateDatabaseName();
+  const dbId = generateUniqueName("db");
   const collectionId = generateUniqueName("col");
   const client = new CosmosClient({
     endpoint: account.documentEndpoint,

test/utils/shared.ts

@@ -4,6 +4,6 @@ export function generateUniqueName(baseName = "", length = 4): string {
   return `${baseName}${crypto.randomBytes(length).toString("hex")}`;
 }
 
-export function generateDatabaseName(baseName = "db", length = 1): string {
+export function generateDatabaseNameWithTimestamp(baseName = "db", length = 1): string {
   return `${baseName}${crypto.randomBytes(length).toString("hex")}-${Date.now()}`;
 }