End to End Test Improvements (#474)

* End to End Test Improvements * indenting * Log completed * Fix up some test * Add delay Co-authored-by: Steve Faulkner <stfaul@microsoft.com>
2021-03-04 18:12:31 -06:00 · 2021-03-04 18:12:31 -06:00 · 498c39c877
parent 87e016f03c
commit 498c39c877
9 changed files with 142 additions and 60 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -144,7 +144,7 @@ jobs:
          NODE_TLS_REJECT_UNAUTHORIZED: 0
  endtoendhosted:
    name: "End to End Tests"
-    needs: [lint, format, compile, unittest]
+    needs: [cleanupaccounts]
    runs-on: ubuntu-latest
    env:
      NODE_TLS_REJECT_UNAUTHORIZED: 0
@ -192,6 +192,21 @@ jobs:
        with:
          name: screenshots
          path: failed-*
+  cleanupaccounts:
+    name: "Cleanup Test Database Accounts"
+    needs: [lint, format, compile, unittest]
+    runs-on: ubuntu-latest
+    env:
+      NOTEBOOKS_TEST_RUNNER_CLIENT_ID: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_ID }}
+      NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET: ${{ secrets.NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: npm ci
+      - run: node utils/cleanupDBs.js
  nuget:
    name: Publish Nuget
    if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/') || contains(github.ref, 'release/')
--- a/package-lock.json
+++ b/package-lock.json
@ -218,6 +218,11 @@
        }
      }
    },
+    "@azure/ms-rest-azure-env": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@azure/ms-rest-azure-env/-/ms-rest-azure-env-2.0.0.tgz",
+      "integrity": "sha512-dG76W7ElfLi+fbTjnZVGj+M9e0BIEJmRxU6fHaUQ12bZBe8EJKYb2GV50YWNaP2uJiVQ5+7nXEVj1VN1UQtaEw=="
+    },
    "@azure/ms-rest-azure-js": {
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/@azure/ms-rest-azure-js/-/ms-rest-azure-js-2.1.0.tgz",
@ -246,6 +251,16 @@
        "xml2js": "^0.4.19"
      }
    },
+    "@azure/ms-rest-nodeauth": {
+      "version": "3.0.7",
+      "resolved": "https://registry.npmjs.org/@azure/ms-rest-nodeauth/-/ms-rest-nodeauth-3.0.7.tgz",
+      "integrity": "sha512-7Q1MyMB+eqUQy8JO+virSIzAjqR2UbKXE/YQZe+53gC8yakm8WOQ5OzGfPP+eyHqeRs6bQESyw2IC5feLWlT2A==",
+      "requires": {
+        "@azure/ms-rest-azure-env": "^2.0.0",
+        "@azure/ms-rest-js": "^2.0.4",
+        "adal-node": "^0.1.28"
+      }
+    },
    "@azure/msal-common": {
      "version": "1.7.2",
      "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-1.7.2.tgz",
@ -5641,6 +5656,38 @@
      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz",
      "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA=="
    },
+    "adal-node": {
+      "version": "0.1.28",
+      "resolved": "https://registry.npmjs.org/adal-node/-/adal-node-0.1.28.tgz",
+      "integrity": "sha1-RoxLs+u9lrEnBmn0ucuk4AZepIU=",
+      "requires": {
+        "@types/node": "^8.0.47",
+        "async": ">=0.6.0",
+        "date-utils": "*",
+        "jws": "3.x.x",
+        "request": ">= 2.52.0",
+        "underscore": ">= 1.3.1",
+        "uuid": "^3.1.0",
+        "xmldom": ">= 0.1.x",
+        "xpath.js": "~1.1.0"
+      },
+      "dependencies": {
+        "@types/node": {
+          "version": "8.10.66",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz",
+          "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw=="
+        },
+        "jws": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
+          "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
+          "requires": {
+            "jwa": "^1.4.1",
+            "safe-buffer": "^5.0.1"
+          }
+        }
+      }
+    },
    "agent-base": {
      "version": "6.0.2",
      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@ -6059,7 +6106,6 @@
      "version": "2.6.3",
      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
-      "dev": true,
      "requires": {
        "lodash": "^4.17.14"
      }
@ -8525,6 +8571,11 @@
      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
      "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
    },
+    "date-utils": {
+      "version": "1.2.21",
+      "resolved": "https://registry.npmjs.org/date-utils/-/date-utils-1.2.21.tgz",
+      "integrity": "sha1-YfsWzcEnSzyayq/+n8ad+HIKK2Q="
+    },
    "dayjs": {
      "version": "1.8.19",
      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.19.tgz",
@ -22584,6 +22635,16 @@
      "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
      "dev": true
    },
+    "xmldom": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.4.0.tgz",
+      "integrity": "sha512-2E93k08T30Ugs+34HBSTQLVtpi6mCddaY8uO+pMNk1pqSjV5vElzn4mmh6KLxN3hki8rNcHSYzILoh3TEWORvA=="
+    },
+    "xpath.js": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/xpath.js/-/xpath.js-1.1.0.tgz",
+      "integrity": "sha512-jg+qkfS4K8E7965sqaUl8mRngXiKb3WZGfONgE18pr03FUQiuSV6G+Ej4tS55B+rIQSFEIw3phdVAQ4pPqNWfQ=="
+    },
    "xregexp": {
      "version": "4.4.1",
      "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.4.1.tgz",
--- a/package.json
+++ b/package.json
@ -8,6 +8,7 @@
    "@azure/cosmos": "3.9.0",
    "@azure/cosmos-language-service": "0.0.5",
    "@azure/identity": "1.2.1",
+    "@azure/ms-rest-nodeauth": "3.0.7",
    "@babel/plugin-proposal-class-properties": "7.12.1",
    "@babel/plugin-proposal-decorators": "7.12.12",
    "@jupyterlab/services": "6.0.2",
--- a/test/mongo/container.spec.ts
+++ b/test/mongo/container.spec.ts
@ -1,6 +1,6 @@
 import "expect-puppeteer";
 import { Frame } from "puppeteer";
-import { generateUniqueName, login } from "../utils/shared";
+import { generateDatabaseName, generateUniqueName, login } from "../utils/shared";

 jest.setTimeout(300000);
 const LOADING_STATE_DELAY = 2500;
@ -11,7 +11,7 @@ const RENDER_DELAY = 1000;
 describe("Collection Add and Delete Mongo spec", () => {
  it("creates a collection", async () => {
    try {
-      const dbId = generateUniqueName("db");
+      const dbId = generateDatabaseName();
      const collectionId = generateUniqueName("col");
      const sharedKey = `${generateUniqueName()}`;
      const frame = await login(process.env.MONGO_CONNECTION_STRING);
--- a/test/mongo/mongoIndexPolicy.spec.ts
+++ b/test/mongo/mongoIndexPolicy.spec.ts
@ -5,7 +5,6 @@ import { generateUniqueName } from "../utils/shared";
 import { ApiKind } from "../../src/Contracts/DataModels";

 const LOADING_STATE_DELAY = 3000;
-const CREATE_DELAY = 5000;
 jest.setTimeout(300000);

 describe("MongoDB Index policy tests", () => {
@ -24,15 +23,16 @@ describe("MongoDB Index policy tests", () => {
      let databases = await frame.$$(`div[class="databaseHeader main1 nodeItem "] > div[class="treeNodeHeader "]`);
      if (databases.length === 0) {
        await createDatabase(frame);
+        await frame.waitFor(25000);
        databases = await frame.$$(`div[class="databaseHeader main1 nodeItem "] > div[class="treeNodeHeader "]`);
      }

-      const selectedDbId = await frame.evaluate((element) => {
-        return element.attributes["data-test"].textContent;
-      }, databases[0]);
+      const selectedDbId = (await frame.evaluate((element) => element.innerText, databases[0]))
+        .replace(/[\u{0080}-\u{FFFF}]/gu, "")
+        .trim();

      // click on database
-      await frame.waitFor(`div[data-test="${selectedDbId}"]`);
+      await frame.waitForSelector(`div[data-test="${selectedDbId}"]`);
      await frame.waitFor(LOADING_STATE_DELAY);
      await frame.click(`div[data-test="${selectedDbId}"]`);
      await frame.waitFor(LOADING_STATE_DELAY);
@ -41,9 +41,9 @@ describe("MongoDB Index policy tests", () => {
      const containers = await frame.$$(
        `div[class="nodeChildren"] > div[class="collectionHeader main2 nodeItem "]> div[class="treeNodeHeader "]`
      );
-      const selectedContainer = await frame.evaluate((element) => {
-        return element.attributes["data-test"].textContent;
-      }, containers[0]);
+      const selectedContainer = (await frame.evaluate((element) => element.innerText, containers[0]))
+        .replace(/[\u{0080}-\u{FFFF}]/gu, "")
+        .trim();
      await frame.waitFor(`div[data-test="${selectedContainer}"]`), { visible: true };
      await frame.waitFor(LOADING_STATE_DELAY);
      await frame.click(`div[data-test="${selectedContainer}"]`);
@ -94,7 +94,7 @@ describe("MongoDB Index policy tests", () => {
          singleFieldIndexInserted = true;
        }
      }
-      await frame.waitFor(LOADING_STATE_DELAY);
+      await frame.waitFor(20000);
      expect(wildCardIndexInserted).toBe(true);
      expect(singleFieldIndexInserted).toBe(true);

@ -107,7 +107,7 @@ describe("MongoDB Index policy tests", () => {
      await onClickSaveButton(frame);

      //check for cleaning
-      await frame.waitFor(CREATE_DELAY);
+      await frame.waitFor(20000);
      await frame.waitFor("div[data-automationid='DetailsRowCell'] > span"), { visible: true };
      const isDeletionComplete = await frame.$$("div[data-automationid='DetailsRowCell'] > span");
      expect(isDeletionComplete).toHaveLength(2);
--- a/test/sql/container.spec.ts
+++ b/test/sql/container.spec.ts
@ -1,6 +1,6 @@
 import "expect-puppeteer";
 import { Frame } from "puppeteer";
-import { generateUniqueName, login } from "../utils/shared";
+import { generateDatabaseName, generateUniqueName, login } from "../utils/shared";

 jest.setTimeout(300000);
 const LOADING_STATE_DELAY = 2500;
@ -11,7 +11,7 @@ const RENDER_DELAY = 1000;
 describe("Collection Add and Delete SQL spec", () => {
  it("creates a collection", async () => {
    try {
-      const dbId = generateUniqueName("db");
+      const dbId = generateDatabaseName();
      const collectionId = generateUniqueName("col");
      const sharedKey = `/skey${generateUniqueName()}`;
      const frame = await login(process.env.PORTAL_RUNNER_CONNECTION_STRING);
--- a/test/sql/resourceToken.spec.ts
+++ b/test/sql/resourceToken.spec.ts
@ -1,7 +1,7 @@
 /* eslint-disable jest/expect-expect */
 import "expect-puppeteer";
 import { Frame } from "puppeteer";
-import { generateUniqueName } from "../utils/shared";
+import { generateDatabaseName, generateUniqueName } from "../utils/shared";
 import { CosmosClient, PermissionMode } from "@azure/cosmos";

 jest.setTimeout(300000);
@ -10,7 +10,7 @@ const CREATE_DELAY = 10000;

 describe("Collection Add and Delete SQL spec", () => {
  it("creates a collection", async () => {
-    const dbId = generateUniqueName("db");
+    const dbId = generateDatabaseName();
    const collectionId = generateUniqueName("col");
    const connectionString = process.env.PORTAL_RUNNER_CONNECTION_STRING;
    const client = new CosmosClient(connectionString);
--- a/test/utils/shared.ts
+++ b/test/utils/shared.ts
@ -26,10 +26,14 @@ export function generateUniqueName(baseName = "", length = 4): string {
  return `${baseName}${crypto.randomBytes(length).toString("hex")}`;
 }

+export function generateDatabaseName(baseName = "db", length = 1): string {
+  return `${baseName}${crypto.randomBytes(length).toString("hex")}-${Date.now()}`;
+}
+
 export async function createDatabase(frame: Frame) {
-  const dbId = generateUniqueName("db");
+  const dbId = generateDatabaseName();
  const collectionId = generateUniqueName("col");
-  const shardKey = generateUniqueName();
+  const shardKey = "partitionKey";
  // create new collection
  await frame.waitFor('button[data-test="New Collection"]', { visible: true });
  await frame.click('button[data-test="New Collection"]');
--- a/utils/cleanupDBs.js
+++ b/utils/cleanupDBs.js
@ -1,51 +1,52 @@
-const { CosmosClient } = require("@azure/cosmos");
+const msRestNodeAuth = require("@azure/ms-rest-nodeauth");
+const { CosmosDBManagementClient } = require("@azure/arm-cosmosdb");

-// TODO: Add support for other API connection strings
-const mongoRegex = RegExp("mongodb://.*:(.*)@(.*).mongo.cosmos.azure.com");
+const clientId = process.env["NOTEBOOKS_TEST_RUNNER_CLIENT_ID"];
+const secret = process.env["NOTEBOOKS_TEST_RUNNER_CLIENT_SECRET"];
+const tenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47";
+const subscriptionId = "69e02f2d-f059-4409-9eac-97e8a276ae2c";
+const resourceGroupName = "runners";

-const connectionString = process.env.PORTAL_RUNNER_CONNECTION_STRING;
-
-async function cleanup() {
-  if (!connectionString) {
-    throw new Error("Connection string not provided");
-  }
-
-  let client;
-  switch (true) {
-    case connectionString.includes("mongodb://"): {
-      const [, key, accountName] = connectionString.match(mongoRegex);
-      client = new CosmosClient({
-        key,
-        endpoint: `https://${accountName}.documents.azure.com:443/`,
-      });
-      break;
-    }
-    // TODO: Add support for other API connection strings
-    default:
-      client = new CosmosClient(connectionString);
-      break;
-  }
-
-  const response = await client.databases.readAll().fetchAll();
-  return Promise.all(
-    response.resources.map(async (db) => {
-      const dbTimestamp = new Date(db._ts * 1000);
 const twentyMinutesAgo = new Date(Date.now() - 1000 * 60 * 20);
-      if (dbTimestamp < twentyMinutesAgo) {
-        await client.database(db.id).delete();
-        console.log(`DELETED: ${db.id} | Timestamp: ${dbTimestamp}`);
+
+// Deletes all SQL and Mongo databases created more than 20 minutes ago in the test runner accounts
+async function main() {
+  const credentials = await msRestNodeAuth.loginWithServicePrincipalSecret(clientId, secret, tenantId);
+  const client = new CosmosDBManagementClient(credentials, subscriptionId);
+  const accounts = await client.databaseAccounts.list(resourceGroupName);
+  for (const account of accounts) {
+    if (account.kind === "MongoDB") {
+      const mongoDatabases = await client.mongoDBResources.listMongoDBDatabases(resourceGroupName, account.name);
+      for (const database of mongoDatabases) {
+        const timestamp = database.name.split("-")[1];
+        if (!timestamp || new Date(timestamp) < twentyMinutesAgo) {
+          await client.mongoDBResources.deleteMongoDBDatabase(resourceGroupName, account.name, database.name);
+          console.log(`DELETED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
        } else {
-        console.log(`SKIPPED: ${db.id} | Timestamp: ${dbTimestamp}`);
+          console.log(`SKIPPED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
+        }
+      }
+    } else if (account.kind === "GlobalDocumentDB") {
+      const sqlDatabases = await client.sqlResources.listSqlDatabases(resourceGroupName, account.name);
+      for (const database of sqlDatabases) {
+        const timestamp = database.name.split("-")[1];
+        if (!timestamp || new Date(timestamp) < twentyMinutesAgo) {
+          await client.sqlResources.deleteSqlDatabase(resourceGroupName, account.name, database.name);
+          console.log(`DELETED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
+        } else {
+          console.log(`SKIPPED: ${account.name} | ${database.name} | Timestamp: ${Date.now()}`);
+        }
+      }
+    }
  }
-    })
-  );
 }

-cleanup()
+main()
  .then(() => {
+    console.log("Completed");
    process.exit(0);
  })
-  .catch((error) => {
-    console.error(error);
+  .catch((err) => {
+    console.error(err);
    process.exit(1);
  });