Protect active E2E test resources from cleanup

package.json

@@ -218,7 +218,7 @@
     "pack:prod": "webpack  --mode production",
     "pack:fast": "webpack  --mode development --progress",
     "copyToConsumers": "node copyToConsumers",
-    "test": "rimraf coverage && jest",
+    "test": "rimraf coverage && jest && node --test utils/cleanupDBs.test.js",
     "test:debug": "jest --runInBand",
     "test:e2e": "jest -c ./jest.config.playwright.js --detectOpenHandles",
     "test:file": "jest --coverage=false",

test/fx.ts

@@ -23,7 +23,10 @@ export function generateUniqueName(baseName: string, options?: TestNameOptions):
   const timestamp = options?.timestampped === undefined ? true : options.timestampped;
   const prefixed = options?.prefixed === undefined ? true : options.prefixed;
 
-  const prefix = prefixed ? "t_" : "";
+  const runId = process.env.GITHUB_RUN_ID;
+  const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? "1";
+  const runPrefix = runId ? `${runId}_${runAttempt}_` : "";
+  const prefix = prefixed ? `t_${runPrefix}` : "";
   const suffix = timestamp ? `_${Date.now()}` : "";
   return `${prefix}${baseName}${crypto.randomBytes(length).toString("hex")}${suffix}`;
 }

utils/cleanupDBs.js

@@ -5,7 +5,15 @@ const ms = require("ms");
 const subscriptionId = process.env["AZURE_SUBSCRIPTION_ID"];
 const resourceGroupName = process.env["E2ETESTS_RESOURCEGROUP_NAME"];
 
-const thirtyMinutesAgo = new Date(Date.now() - 1000 * 60 * 30).getTime();
+const cleanupMinimumAge = ms(process.env["E2E_CLEANUP_MINIMUM_AGE"] || "6h");
+if (!cleanupMinimumAge) {
+  throw new Error("E2E_CLEANUP_MINIMUM_AGE must be a valid duration");
+}
+const cleanupThreshold = Date.now() - cleanupMinimumAge;
+
+function shouldDeleteResource(name, timestamp, threshold = cleanupThreshold) {
+  return Boolean(name?.startsWith("t_") && timestamp && timestamp < threshold);
+}
 
 function friendlyTime(date) {
   try {
@@ -29,22 +37,27 @@ async function main() {
       for await (const database of mongoDatabases) {
         // Unfortunately Mongo does not provide a timestamp in ARM. There is no way to tell how old the DB is other thn encoding it in the ID :(
         const timestamp = Number(database.name.split("_").pop());
-        if (timestamp && timestamp < thirtyMinutesAgo) {
+        if (shouldDeleteResource(database.name, timestamp)) {
-          await client.mongoDBResources.beginDeleteMongoDBDatabaseAndWait(resourceGroupName, account.name, database.name);
+          await client.mongoDBResources.beginDeleteMongoDBDatabaseAndWait(
+            resourceGroupName,
+            account.name,
+            database.name,
+          );
           console.log(`DELETED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
         } else {
           console.log(`SKIPPED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
         }
       }
     } else if (account.capabilities.find((c) => c.name === "EnableCassandra")) {
-      const cassandraDatabases = client.cassandraResources.listCassandraKeyspaces(
+      const cassandraDatabases = client.cassandraResources.listCassandraKeyspaces(resourceGroupName, account.name);
-        resourceGroupName,
-        account.name,
-      );
       for await (const database of cassandraDatabases) {
         const timestamp = Number(database.resource.ts) * 1000;
-        if (timestamp && timestamp < thirtyMinutesAgo) {
+        if (shouldDeleteResource(database.name, timestamp)) {
-          await client.cassandraResources.beginDeleteCassandraKeyspaceAndWait(resourceGroupName, account.name, database.name);
+          await client.cassandraResources.beginDeleteCassandraKeyspaceAndWait(
+            resourceGroupName,
+            account.name,
+            database.name,
+          );
           console.log(`DELETED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
         } else {
           console.log(`SKIPPED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
@@ -54,7 +67,7 @@ async function main() {
       const tablesDatabase = client.tableResources.listTables(resourceGroupName, account.name);
       for await (const database of tablesDatabase) {
         const timestamp = Number(database.resource.ts) * 1000;
-        if (timestamp && timestamp < thirtyMinutesAgo) {
+        if (shouldDeleteResource(database.name, timestamp)) {
           await client.tableResources.beginDeleteTableAndWait(resourceGroupName, account.name, database.name);
           console.log(`DELETED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
         } else {
@@ -65,8 +78,12 @@ async function main() {
       const graphDatabases = client.gremlinResources.listGremlinDatabases(resourceGroupName, account.name);
       for await (const database of graphDatabases) {
         const timestamp = Number(database.resource.ts) * 1000;
-        if (timestamp && timestamp < thirtyMinutesAgo) {
+        if (shouldDeleteResource(database.name, timestamp)) {
-          await client.gremlinResources.beginDeleteGremlinDatabaseAndWait(resourceGroupName, account.name, database.name);
+          await client.gremlinResources.beginDeleteGremlinDatabaseAndWait(
+            resourceGroupName,
+            account.name,
+            database.name,
+          );
           console.log(`DELETED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
         } else {
           console.log(`SKIPPED: ${account.name} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
@@ -92,7 +109,7 @@ async function deleteWithRetry(client, database, accountName) {
   while (attempt < maxRetries) {
     try {
       const timestamp = Number(database.resource.ts) * 1000;
-      if (timestamp && timestamp < thirtyMinutesAgo) {
+      if (shouldDeleteResource(database.name, timestamp)) {
         await client.sqlResources.beginDeleteSqlDatabaseAndWait(resourceGroupName, accountName, database.name);
         console.log(`DELETED: ${accountName} | ${database.name} | Age: ${friendlyTime(Date.now() - timestamp)}`);
       } else {
@@ -118,9 +135,10 @@ async function deleteWithRetry(client, database, accountName) {
 
 // Helper function to delay the retry attempts
 function delay(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+if (require.main === module) {
   main()
     .then(() => {
       console.log("Completed");
@@ -130,3 +148,6 @@ main()
       console.error(err);
       process.exit(1);
     });
+}
+
+module.exports = { shouldDeleteResource };

utils/cleanupDBs.test.js

@@ -0,0 +1,23 @@
+const assert = require("node:assert/strict");
+const test = require("node:test");
+const { shouldDeleteResource } = require("./cleanupDBs");
+
+const cleanupThreshold = Date.now();
+
+test("deletes owned test resources older than the threshold", () => {
+  assert.equal(shouldDeleteResource("t_12345_1_dbab_1000", cleanupThreshold - 1, cleanupThreshold), true);
+});
+
+test("keeps owned test resources at or newer than the threshold", () => {
+  assert.equal(shouldDeleteResource("t_12345_1_dbab_1000", cleanupThreshold, cleanupThreshold), false);
+  assert.equal(shouldDeleteResource("t_12345_1_dbab_1000", cleanupThreshold + 1, cleanupThreshold), false);
+});
+
+test("keeps resources that are not marked as test-owned", () => {
+  assert.equal(shouldDeleteResource("seeded-database", cleanupThreshold - 1, cleanupThreshold), false);
+});
+
+test("keeps resources without a valid name or timestamp", () => {
+  assert.equal(shouldDeleteResource(undefined, cleanupThreshold - 1, cleanupThreshold), false);
+  assert.equal(shouldDeleteResource("t_12345_1_dbab_1000", undefined, cleanupThreshold), false);
+});