End to End Test Improvements (#474)

* End to End Test Improvements

* indenting

* Log completed

* Fix up some test

* Add delay

Co-authored-by: Steve Faulkner <stfaul@microsoft.com>

.github/workflows/ci.yml

@@ -144,7 +144,7 @@ jobs:
           NODE_TLS_REJECT_UNAUTHORIZED: 0
   endtoendhosted:
     name: "End to End Tests"
-    needs: [lint, format, compile, unittest]
+    needs: [cleanupaccounts]
     runs-on: ubuntu-latest
     env:
       NODE_TLS_REJECT_UNAUTHORIZED: 0
@@ -192,6 +192,21 @@ jobs:
         with:
           name: screenshots
           path: failed-*
+  cleanupaccounts:
+    name: "Cleanup Test Database Accounts"
+    needs: [lint, format, compile, unittest]
+    runs-on: ubuntu-latest
+    env:
+      NOTEBOOKS_TEST_RUNNER_CLIENT_ID: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_ID }}
+      NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: npm ci
+      - run: node utils/cleanupDBs.js
   nuget:
     name: Publish Nuget
     if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/') || contains(github.ref, 'release/')

package-lock.json

@@ -218,6 +218,11 @@
         }
       }
     },
+    "@azure/ms-rest-azure-env": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@azure/ms-rest-azure-env/-/ms-rest-azure-env-2.0.0.tgz",
+      "integrity": "sha512-dG76W7ElfLi+fbTjnZVGj+M9e0BIEJmRxU6fHaUQ12bZBe8EJKYb2GV50YWNaP2uJiVQ5+7nXEVj1VN1UQtaEw=="
+    },
     "@azure/ms-rest-azure-js": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/@azure/ms-rest-azure-js/-/ms-rest-azure-js-2.1.0.tgz",
@@ -246,6 +251,16 @@
         "xml2js": "^0.4.19"
       }
     },
+    "@azure/ms-rest-nodeauth": {
+      "version": "3.0.7",
+      "resolved": "https://registry.npmjs.org/@azure/ms-rest-nodeauth/-/ms-rest-nodeauth-3.0.7.tgz",
+      "integrity": "sha512-7Q1MyMB+eqUQy8JO+virSIzAjqR2UbKXE/YQZe+53gC8yakm8WOQ5OzGfPP+eyHqeRs6bQESyw2IC5feLWlT2A==",
+      "requires": {
+        "@azure/ms-rest-azure-env": "^2.0.0",
+        "@azure/ms-rest-js": "^2.0.4",
+        "adal-node": "^0.1.28"
+      }
+    },
     "@azure/msal-common": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-1.7.2.tgz",
@@ -5641,6 +5656,38 @@
       "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz",
       "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA=="
     },
+    "adal-node": {
+      "version": "0.1.28",
+      "resolved": "https://registry.npmjs.org/adal-node/-/adal-node-0.1.28.tgz",
+      "integrity": "sha1-RoxLs+u9lrEnBmn0ucuk4AZepIU=",
+      "requires": {
+        "@types/node": "^8.0.47",
+        "async": ">=0.6.0",
+        "date-utils": "*",
+        "jws": "3.x.x",
+        "request": ">= 2.52.0",
+        "underscore": ">= 1.3.1",
+        "uuid": "^3.1.0",
+        "xmldom": ">= 0.1.x",
+        "xpath.js": "~1.1.0"
+      },
+      "dependencies": {
+        "@types/node": {
+          "version": "8.10.66",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz",
+          "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw=="
+        },
+        "jws": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
+          "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
+          "requires": {
+            "jwa": "^1.4.1",
+            "safe-buffer": "^5.0.1"
+          }
+        }
+      }
+    },
     "agent-base": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@@ -6059,7 +6106,6 @@
       "version": "2.6.3",
       "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
       "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
-      "dev": true,
       "requires": {
         "lodash": "^4.17.14"
       }
@@ -8525,6 +8571,11 @@
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
       "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
     },
+    "date-utils": {
+      "version": "1.2.21",
+      "resolved": "https://registry.npmjs.org/date-utils/-/date-utils-1.2.21.tgz",
+      "integrity": "sha1-YfsWzcEnSzyayq/+n8ad+HIKK2Q="
+    },
     "dayjs": {
       "version": "1.8.19",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.19.tgz",
@@ -22584,6 +22635,16 @@
       "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
       "dev": true
     },
+    "xmldom": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.4.0.tgz",
+      "integrity": "sha512-2E93k08T30Ugs+34HBSTQLVtpi6mCddaY8uO+pMNk1pqSjV5vElzn4mmh6KLxN3hki8rNcHSYzILoh3TEWORvA=="
+    },
+    "xpath.js": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/xpath.js/-/xpath.js-1.1.0.tgz",
+      "integrity": "sha512-jg+qkfS4K8E7965sqaUl8mRngXiKb3WZGfONgE18pr03FUQiuSV6G+Ej4tS55B+rIQSFEIw3phdVAQ4pPqNWfQ=="
+    },
     "xregexp": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.4.1.tgz",

package.json

@@ -8,6 +8,7 @@
     "@azure/cosmos": "3.9.0",
     "@azure/cosmos-language-service": "0.0.5",
     "@azure/identity": "1.2.1",
+    "@azure/ms-rest-nodeauth": "3.0.7",
     "@babel/plugin-proposal-class-properties": "7.12.1",
     "@babel/plugin-proposal-decorators": "7.12.12",
     "@jupyterlab/services": "6.0.2",

test/mongo/container.spec.ts

@@ -1,6 +1,6 @@
 import "expect-puppeteer";
 import { Frame } from "puppeteer";
-import { generateUniqueName, login } from "../utils/shared";
+import { generateDatabaseName, generateUniqueName, login } from "../utils/shared";
 
 jest.setTimeout(300000);
 const LOADING_STATE_DELAY = 2500;
@@ -11,7 +11,7 @@ const RENDER_DELAY = 1000;
 describe("Collection Add and Delete Mongo spec", () => {
   it("creates a collection", async () => {
     try {
-      const dbId = generateUniqueName("db");
+      const dbId = generateDatabaseName();
       const collectionId = generateUniqueName("col");
       const sharedKey = `${generateUniqueName()}`;
       const frame = await login(process.env.MONGO_CONNECTION_STRING);

test/mongo/mongoIndexPolicy.spec.ts

@@ -5,7 +5,6 @@ import { generateUniqueName } from "../utils/shared";
 import { ApiKind } from "../../src/Contracts/DataModels";
 
 const LOADING_STATE_DELAY = 3000;
-const CREATE_DELAY = 5000;
 jest.setTimeout(300000);
 
 describe("MongoDB Index policy tests", () => {
@@ -24,15 +23,16 @@ describe("MongoDB Index policy tests", () => {
       let databases = await frame.$$(`div[class="databaseHeader main1 nodeItem "] > div[class="treeNodeHeader "]`);
       if (databases.length === 0) {
         await createDatabase(frame);
+        await frame.waitFor(25000);
         databases = await frame.$$(`div[class="databaseHeader main1 nodeItem "] > div[class="treeNodeHeader "]`);
       }
 
-      const selectedDbId = await frame.evaluate((element) => {
+      const selectedDbId = (await frame.evaluate((element) => element.innerText, databases[0]))
-        return element.attributes["data-test"].textContent;
+        .replace(/[\u{0080}-\u{FFFF}]/gu, "")
-      }, databases[0]);
+        .trim();
 
       // click on database
-      await frame.waitFor(`div[data-test="${selectedDbId}"]`);
+      await frame.waitForSelector(`div[data-test="${selectedDbId}"]`);
       await frame.waitFor(LOADING_STATE_DELAY);
       await frame.click(`div[data-test="${selectedDbId}"]`);
       await frame.waitFor(LOADING_STATE_DELAY);
@@ -41,9 +41,9 @@ describe("MongoDB Index policy tests", () => {
       const containers = await frame.$$(
         `div[class="nodeChildren"] > div[class="collectionHeader main2 nodeItem "]> div[class="treeNodeHeader "]`
       );
-      const selectedContainer = await frame.evaluate((element) => {
+      const selectedContainer = (await frame.evaluate((element) => element.innerText, containers[0]))
-        return element.attributes["data-test"].textContent;
+        .replace(/[\u{0080}-\u{FFFF}]/gu, "")
-      }, containers[0]);
+        .trim();
       await frame.waitFor(`div[data-test="${selectedContainer}"]`), { visible: true };
       await frame.waitFor(LOADING_STATE_DELAY);
       await frame.click(`div[data-test="${selectedContainer}"]`);
@@ -94,7 +94,7 @@ describe("MongoDB Index policy tests", () => {
           singleFieldIndexInserted = true;
         }
       }
-      await frame.waitFor(LOADING_STATE_DELAY);
+      await frame.waitFor(20000);
       expect(wildCardIndexInserted).toBe(true);
       expect(singleFieldIndexInserted).toBe(true);
 
@@ -107,7 +107,7 @@ describe("MongoDB Index policy tests", () => {
       await onClickSaveButton(frame);
 
       //check for cleaning
-      await frame.waitFor(CREATE_DELAY);
+      await frame.waitFor(20000);
       await frame.waitFor("div[data-automationid='DetailsRowCell'] > span"), { visible: true };
       const isDeletionComplete = await frame.$$("div[data-automationid='DetailsRowCell'] > span");
       expect(isDeletionComplete).toHaveLength(2);

test/sql/container.spec.ts

@@ -1,6 +1,6 @@
 import "expect-puppeteer";
 import { Frame } from "puppeteer";
-import { generateUniqueName, login } from "../utils/shared";
+import { generateDatabaseName, generateUniqueName, login } from "../utils/shared";
 
 jest.setTimeout(300000);
 const LOADING_STATE_DELAY = 2500;
@@ -11,7 +11,7 @@ const RENDER_DELAY = 1000;
 describe("Collection Add and Delete SQL spec", () => {
   it("creates a collection", async () => {
     try {
-      const dbId = generateUniqueName("db");
+      const dbId = generateDatabaseName();
       const collectionId = generateUniqueName("col");
       const sharedKey = `/skey${generateUniqueName()}`;
       const frame = await login(process.env.PORTAL_RUNNER_CONNECTION_STRING);

test/sql/resourceToken.spec.ts

@@ -1,7 +1,7 @@
 /* eslint-disable jest/expect-expect */
 import "expect-puppeteer";
 import { Frame } from "puppeteer";
-import { generateUniqueName } from "../utils/shared";
+import { generateDatabaseName, generateUniqueName } from "../utils/shared";
 import { CosmosClient, PermissionMode } from "@azure/cosmos";
 
 jest.setTimeout(300000);
@@ -10,7 +10,7 @@ const CREATE_DELAY = 10000;
 
 describe("Collection Add and Delete SQL spec", () => {
   it("creates a collection", async () => {
-    const dbId = generateUniqueName("db");
+    const dbId = generateDatabaseName();
     const collectionId = generateUniqueName("col");
     const connectionString = process.env.PORTAL_RUNNER_CONNECTION_STRING;
     const client = new CosmosClient(connectionString);

test/utils/shared.ts

@@ -26,10 +26,14 @@ export function generateUniqueName(baseName = "", length = 4): string {
   return `${baseName}${crypto.randomBytes(length).toString("hex")}`;
 }
 
+export function generateDatabaseName(baseName = "db", length = 1): string {
+  return `${baseName}${crypto.randomBytes(length).toString("hex")}-${Date.now()}`;
+}
+
 export async function createDatabase(frame: Frame) {
-  const dbId = generateUniqueName("db");
+  const dbId = generateDatabaseName();
   const collectionId = generateUniqueName("col");
-  const shardKey = generateUniqueName();
+  const shardKey = "partitionKey";
   // create new collection
   await frame.waitFor('button[data-test="New Collection"]', { visible: true });
   await frame.click('button[data-test="New Collection"]');

utils/cleanupDBs.js

@@ -1,51 +1,52 @@
-const { CosmosClient } = require("@azure/cosmos");
+const msRestNodeAuth = require("@azure/ms-rest-nodeauth");
+const { CosmosDBManagementClient } = require("@azure/arm-cosmosdb");
 
-// TODO: Add support for other API connection strings
+const clientId = process.env["NOTEBOOKS_TEST_RUNNER_CLIENT_ID"];
-const mongoRegex = RegExp("mongodb://.*:(.*)@(.*).mongo.cosmos.azure.com");
+const secret = process.env["NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET"];
+const tenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47";
+const subscriptionId = "69e02f2d-f059-4409-9eac-97e8a276ae2c";
+const resourceGroupName = "runners";
 
-const connectionString = process.env.PORTAL_RUNNER_CONNECTION_STRING;
+const twentyMinutesAgo = new Date(Date.now() - 1000 * 60 * 20);
 
-async function cleanup() {
+// Deletes all SQL and Mongo databases created more than 20 minutes ago in the test runner accounts
-  if (!connectionString) {
+async function main() {
-    throw new Error("Connection string not provided");
+  const credentials = await msRestNodeAuth.loginWithServicePrincipalSecret(clientId, secret, tenantId);
-  }
+  const client = new CosmosDBManagementClient(credentials, subscriptionId);
-
+  const accounts = await client.databaseAccounts.list(resourceGroupName);
-  let client;
+  for (const account of accounts) {
-  switch (true) {
+    if (account.kind === "MongoDB") {
-    case connectionString.includes("mongodb://"): {
+      const mongoDatabases = await client.mongoDBResources.listMongoDBDatabases(resourceGroupName, account.name);
-      const [, key, accountName] = connectionString.match(mongoRegex);
+      for (const database of mongoDatabases) {
-      client = new CosmosClient({
+        const timestamp = database.name.split("-")[1];
-        key,
+        if (!timestamp || new Date(timestamp) < twentyMinutesAgo) {
-        endpoint: `https://${accountName}.documents.azure.com:443/`,
+          await client.mongoDBResources.deleteMongoDBDatabase(resourceGroupName, account.name, database.name);
-      });
+          console.log(`DELETED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
-      break;
-    }
-    // TODO: Add support for other API connection strings
-    default:
-      client = new CosmosClient(connectionString);
-      break;
-  }
-
-  const response = await client.databases.readAll().fetchAll();
-  return Promise.all(
-    response.resources.map(async (db) => {
-      const dbTimestamp = new Date(db._ts * 1000);
-      const twentyMinutesAgo = new Date(Date.now() - 1000 * 60 * 20);
-      if (dbTimestamp < twentyMinutesAgo) {
-        await client.database(db.id).delete();
-        console.log(`DELETED: ${db.id} | Timestamp: ${dbTimestamp}`);
         } else {
-        console.log(`SKIPPED: ${db.id} | Timestamp: ${dbTimestamp}`);
+          console.log(`SKIPPED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
+        }
+      }
+    } else if (account.kind === "GlobalDocumentDB") {
+      const sqlDatabases = await client.sqlResources.listSqlDatabases(resourceGroupName, account.name);
+      for (const database of sqlDatabases) {
+        const timestamp = database.name.split("-")[1];
+        if (!timestamp || new Date(timestamp) < twentyMinutesAgo) {
+          await client.sqlResources.deleteSqlDatabase(resourceGroupName, account.name, database.name);
+          console.log(`DELETED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
+        } else {
+          console.log(`SKIPPED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
+        }
+      }
+    }
   }
-    })
-  );
 }
 
-cleanup()
+main()
   .then(() => {
+    console.log("Completed");
     process.exit(0);
   })
-  .catch((error) => {
+  .catch((err) => {
-    console.error(error);
+    console.error(err);
     process.exit(1);
   });