Pk missing fix (#2107)

* fix partition key missing not being able to load the document

* Implement E2E tests for documents with different partitionkeys

* Implement E2E tests for documents with different partitionkeys

* Implement E2E tests for documents with different partitionkeys

* Updated snapshot

* Updated tests for MongoRU and add create/delete tests

* Fixing system partition key showing up in Data Explorer

test/fx.ts

@@ -1,5 +1,5 @@
 import { AzureCliCredential } from "@azure/identity";
-import { expect, Frame, Locator, Page } from "@playwright/test";
+import { Frame, Locator, Page, expect } from "@playwright/test";
 import crypto from "crypto";
 
 const RETRY_COUNT = 3;
@@ -26,7 +26,7 @@ export function getAzureCLICredentials(): AzureCliCredential {
 
 export async function getAzureCLICredentialsToken(): Promise<string> {
   const credentials = getAzureCLICredentials();
-  const token = (await credentials.getToken("https://management.core.windows.net//.default")).token;
+  const token = (await credentials.getToken("https://management.core.windows.net//.default"))?.token || "";
   return token;
 }
 
@@ -35,8 +35,10 @@ export enum TestAccount {
   Cassandra = "Cassandra",
   Gremlin = "Gremlin",
   Mongo = "Mongo",
+  MongoReadonly = "MongoReadOnly",
   Mongo32 = "Mongo32",
   SQL = "SQL",
+  SQLReadOnly = "SQLReadOnly",
 }
 
 export const defaultAccounts: Record<TestAccount, string> = {
@@ -44,8 +46,10 @@ export const defaultAccounts: Record<TestAccount, string> = {
   [TestAccount.Cassandra]: "github-e2etests-cassandra",
   [TestAccount.Gremlin]: "github-e2etests-gremlin",
   [TestAccount.Mongo]: "github-e2etests-mongo",
+  [TestAccount.MongoReadonly]: "github-e2etests-mongo-readonly",
   [TestAccount.Mongo32]: "github-e2etests-mongo32",
   [TestAccount.SQL]: "github-e2etests-sql",
+  [TestAccount.SQLReadOnly]: "github-e2etests-sql-readonly",
 };
 
 export const resourceGroupName = process.env.DE_TEST_RESOURCE_GROUP ?? "de-e2e-tests";
@@ -214,6 +218,25 @@ export class QueryTab {
   }
 }
 
+export class DocumentsTab {
+  documentsFilter: Locator;
+  documentsListPane: Locator;
+  documentResultsPane: Locator;
+  resultsEditor: Editor;
+
+  constructor(
+    public frame: Frame,
+    public tabId: string,
+    public tab: Locator,
+    public locator: Locator,
+  ) {
+    this.documentsFilter = this.locator.getByTestId("DocumentsTab/Filter");
+    this.documentsListPane = this.locator.getByTestId("DocumentsTab/DocumentsPane");
+    this.documentResultsPane = this.locator.getByTestId("DocumentsTab/ResultsPane");
+    this.resultsEditor = new Editor(this.frame, this.documentResultsPane.getByTestId("EditorReact/Host/Loaded"));
+  }
+}
+
 type PanelOpenOptions = {
   closeTimeout?: number;
 };
@@ -232,6 +255,12 @@ export class DataExplorer {
     return new QueryTab(this.frame, tabId, tab, queryTab);
   }
 
+  documentsTab(tabId: string): DocumentsTab {
+    const tab = this.tab(tabId);
+    const documentsTab = tab.getByTestId("DocumentsTab");
+    return new DocumentsTab(this.frame, tabId, tab, documentsTab);
+  }
+
   /** Select the primary global command button.
    *
    * There's only a single "primary" button, but we still require you to pass the label to confirm you're selecting the right button.
@@ -245,6 +274,10 @@ export class DataExplorer {
     return this.frame.getByTestId(`CommandBar/Button:${label}`).and(this.frame.locator("css=button"));
   }
 
+  dialogButton(label: string): Locator {
+    return this.frame.getByTestId(`DialogButton:${label}`).and(this.frame.locator("css=button"));
+  }
+
   /** Select the side panel with the specified title */
   panel(title: string): Locator {
     return this.frame.getByTestId(`Panel:${title}`);
@@ -294,6 +327,26 @@ export class DataExplorer {
     return await this.waitForNode(`${databaseId}/${containerId}`);
   }
 
+  async waitForContainerItemsNode(databaseId: string, containerId: string): Promise<TreeNode> {
+    return await this.waitForNode(`${databaseId}/${containerId}/Items`);
+  }
+
+  async waitForContainerDocumentsNode(databaseId: string, containerId: string): Promise<TreeNode> {
+    return await this.waitForNode(`${databaseId}/${containerId}/Documents`);
+  }
+
+  async waitForCommandBarButton(label: string, timeout?: number): Promise<Locator> {
+    const commandBar = this.commandBarButton(label);
+    await commandBar.waitFor({ state: "visible", timeout });
+    return commandBar;
+  }
+
+  async waitForDialogButton(label: string, timeout?: number): Promise<Locator> {
+    const dialogButton = this.dialogButton(label);
+    await dialogButton.waitFor({ timeout });
+    return dialogButton;
+  }
+
   /** Select the tree node with the specified id */
   treeNode(id: string): TreeNode {
     return new TreeNode(this.frame.getByTestId(`TreeNode:${id}`), this.frame, id);

test/mongo/document.spec.ts

@@ -0,0 +1,89 @@
+import { expect, test } from "@playwright/test";
+
+import { DataExplorer, DocumentsTab, TestAccount } from "../fx";
+import { retry, serializeMongoToJson, setPartitionKeys } from "../testData";
+import { documentTestCases } from "./testCases";
+
+let explorer: DataExplorer = null!;
+let documentsTab: DocumentsTab = null!;
+
+for (const { name, databaseId, containerId, documents } of documentTestCases) {
+  test.describe(`Test MongoRU Documents with ${name}`, () => {
+    test.beforeEach("Open documents tab", async ({ page }) => {
+      explorer = await DataExplorer.open(page, TestAccount.MongoReadonly);
+
+      const containerNode = await explorer.waitForContainerNode(databaseId, containerId);
+      await containerNode.expand();
+
+      const containerMenuNode = await explorer.waitForContainerDocumentsNode(databaseId, containerId);
+      await containerMenuNode.element.click();
+
+      documentsTab = explorer.documentsTab("tab0");
+
+      await documentsTab.documentsFilter.waitFor();
+      await documentsTab.documentsListPane.waitFor();
+      await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+    });
+
+    for (const document of documents) {
+      const { documentId: docId, partitionKeys } = document;
+      test.describe(`Document ID: ${docId}`, () => {
+        test(`should load and view document ${docId}`, async () => {
+          const span = documentsTab.documentsListPane.getByText(docId, { exact: true }).nth(0);
+          await span.waitFor();
+          await expect(span).toBeVisible();
+
+          await span.click();
+          await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+
+          const resultText = await documentsTab.resultsEditor.text();
+          const resultData = serializeMongoToJson(resultText!);
+          expect(resultText).not.toBeNull();
+          expect(resultData?._id).not.toBeNull();
+          expect(resultData?._id).toEqual(docId);
+        });
+        test(`should be able to create and delete new document from ${docId}`, async () => {
+          const span = documentsTab.documentsListPane.getByText(docId, { exact: true }).nth(0);
+          await span.waitFor();
+          await expect(span).toBeVisible();
+
+          await span.click();
+          let newDocumentId;
+          await retry(async () => {
+            const newDocumentButton = await explorer.waitForCommandBarButton("New Document", 5000);
+            await expect(newDocumentButton).toBeVisible();
+            await expect(newDocumentButton).toBeEnabled();
+            await newDocumentButton.click();
+
+            await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+
+            newDocumentId = `${Date.now().toString()}-delete`;
+
+            const newDocument = {
+              _id: newDocumentId,
+              ...setPartitionKeys(partitionKeys || []),
+            };
+
+            await documentsTab.resultsEditor.setText(JSON.stringify(newDocument));
+            const saveButton = await explorer.waitForCommandBarButton("Save", 5000);
+            await saveButton.click({ timeout: 5000 });
+          }, 3);
+
+          const newSpan = documentsTab.documentsListPane.getByText(newDocumentId, { exact: true }).nth(0);
+          await newSpan.waitFor();
+          await newSpan.click();
+          await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+
+          const deleteButton = await explorer.waitForCommandBarButton("Delete", 5000);
+          await deleteButton.click();
+
+          const deleteDialogButton = await explorer.waitForDialogButton("Delete", 5000);
+          await deleteDialogButton.click();
+
+          const deletedSpan = documentsTab.documentsListPane.getByText(newDocumentId, { exact: true }).nth(0);
+          await expect(deletedSpan).toHaveCount(0);
+        });
+      });
+    }
+  });
+}

test/mongo/testCases.ts

@@ -0,0 +1,31 @@
+import { DocumentTestCase } from "../testData";
+
+export const documentTestCases: DocumentTestCase[] = [
+  {
+    name: "Unsharded Collection",
+    databaseId: "e2etests-mongo-readonly",
+    containerId: "unsharded",
+    documents: [
+      {
+        documentId: "unsharded",
+        partitionKeys: [],
+      },
+    ],
+  },
+  {
+    name: "Sharded Collection",
+    databaseId: "e2etests-mongo-readonly",
+    containerId: "sharded",
+    documents: [
+      {
+        documentId: "sharded",
+        partitionKeys: [
+          {
+            key: "/shardKey",
+            value: "shardKey",
+          },
+        ],
+      },
+    ],
+  },
+];

test/sql/document.spec.ts

@@ -0,0 +1,93 @@
+import { expect, test } from "@playwright/test";
+
+import { DataExplorer, DocumentsTab, TestAccount } from "../fx";
+import { retry, setPartitionKeys } from "../testData";
+import { documentTestCases } from "./testCases";
+
+let explorer: DataExplorer = null!;
+let documentsTab: DocumentsTab = null!;
+
+for (const { name, databaseId, containerId, documents } of documentTestCases) {
+  test.describe(`Test SQL Documents with ${name}`, () => {
+    test.beforeEach("Open documents tab", async ({ page }) => {
+      explorer = await DataExplorer.open(page, TestAccount.SQLReadOnly);
+
+      const containerNode = await explorer.waitForContainerNode(databaseId, containerId);
+      await containerNode.expand();
+
+      const containerMenuNode = await explorer.waitForContainerItemsNode(databaseId, containerId);
+      await containerMenuNode.element.click();
+
+      documentsTab = explorer.documentsTab("tab0");
+
+      await documentsTab.documentsFilter.waitFor();
+      await documentsTab.documentsListPane.waitFor();
+      await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+    });
+
+    for (const document of documents) {
+      const { documentId: docId, partitionKeys } = document;
+      test.describe(`Document ID: ${docId}`, () => {
+        test(`should load and view document ${docId}`, async () => {
+          const span = documentsTab.documentsListPane.getByText(docId, { exact: true }).nth(0);
+          await span.waitFor();
+          await expect(span).toBeVisible();
+
+          await span.click();
+          await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+
+          const resultText = await documentsTab.resultsEditor.text();
+          const resultData = JSON.parse(resultText!);
+          expect(resultText).not.toBeNull();
+          expect(resultData?.id).toEqual(docId);
+        });
+        test(`should be able to create and delete new document from ${docId}`, async ({ page }) => {
+          const span = documentsTab.documentsListPane.getByText(docId, { exact: true }).nth(0);
+          await span.waitFor();
+          await expect(span).toBeVisible();
+
+          await span.click();
+          let newDocumentId;
+          await page.waitForTimeout(5000);
+          await retry(async () => {
+            // const discardButton = await explorer.waitForCommandBarButton("Discard", 5000);
+            // if (await discardButton.isEnabled()) {
+            //   await discardButton.click();
+            // }
+            const newDocumentButton = await explorer.waitForCommandBarButton("New Item", 5000);
+            await expect(newDocumentButton).toBeVisible();
+            await expect(newDocumentButton).toBeEnabled();
+            await newDocumentButton.click();
+
+            await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+
+            newDocumentId = `${Date.now().toString()}-delete`;
+
+            const newDocument = {
+              id: newDocumentId,
+              ...setPartitionKeys(partitionKeys || []),
+            };
+
+            await documentsTab.resultsEditor.setText(JSON.stringify(newDocument));
+            const saveButton = await explorer.waitForCommandBarButton("Save", 5000);
+            await saveButton.click({ timeout: 5000 });
+          }, 3);
+
+          const newSpan = documentsTab.documentsListPane.getByText(newDocumentId, { exact: true }).nth(0);
+          await newSpan.waitFor();
+          await newSpan.click();
+          await expect(documentsTab.resultsEditor.locator).toBeAttached({ timeout: 60 * 1000 });
+
+          const deleteButton = await explorer.waitForCommandBarButton("Delete", 5000);
+          await deleteButton.click();
+
+          const deleteDialogButton = await explorer.waitForDialogButton("Delete", 5000);
+          await deleteDialogButton.click();
+
+          const deletedSpan = documentsTab.documentsListPane.getByText(newDocumentId, { exact: true }).nth(0);
+          await expect(deletedSpan).toHaveCount(0);
+        });
+      });
+    }
+  });
+}

test/sql/testCases.ts

@@ -0,0 +1,235 @@
+import { DocumentTestCase } from "../testData";
+
+export const documentTestCases: DocumentTestCase[] = [
+  {
+    name: "System Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "systemPartitionKey",
+    documents: [{ documentId: "systempartition", partitionKeys: [] }],
+  },
+  {
+    name: "Single Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "singlePartitionKey",
+    documents: [
+      {
+        documentId: "singlePartitionKey",
+        partitionKeys: [{ key: "/singlePartitionKey", value: "singlePartitionKey" }],
+      },
+      {
+        documentId: "singlePartitionKey_empty_string",
+        partitionKeys: [{ key: "/singlePartitionKey", value: "" }],
+      },
+      {
+        documentId: "singlePartitionKey_null",
+        partitionKeys: [{ key: "/singlePartitionKey", value: null }],
+      },
+      {
+        documentId: "singlePartitionKey_missing",
+        partitionKeys: [],
+      },
+    ],
+  },
+  {
+    name: "Single Nested Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "singleNestedPartitionKey",
+    documents: [
+      {
+        documentId: "singlePartitionKey_nested",
+        partitionKeys: [{ key: "/singlePartitionKey/nested", value: "nestedValue" }],
+      },
+      {
+        documentId: "singlePartitionKey_nested_empty_string",
+        partitionKeys: [{ key: "/singlePartitionKey/nested", value: "" }],
+      },
+      {
+        documentId: "singlePartitionKey_nested_null",
+        partitionKeys: [{ key: "/singlePartitionKey/nested", value: null }],
+      },
+      {
+        documentId: "singlePartitionKey_nested_missing",
+        partitionKeys: [],
+      },
+    ],
+  },
+  {
+    name: "2-Level Hierarchical Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "twoLevelPartitionKey",
+    documents: [
+      {
+        documentId: "twoLevelPartitionKey_value_empty",
+        partitionKeys: [
+          { key: "/twoLevelPartitionKey_1", value: "value" },
+          { key: "/twoLevelPartitionKey_2", value: "" },
+        ],
+      },
+      {
+        documentId: "twoLevelPartitionKey_value_null",
+        partitionKeys: [
+          { key: "/twoLevelPartitionKey_1", value: "value" },
+          { key: "/twoLevelPartitionKey_2", value: null },
+        ],
+      },
+      {
+        documentId: "twoLevelPartitionKey_value_missing",
+        partitionKeys: [{ key: "/twoLevelPartitionKey_1", value: "value" }],
+      },
+      {
+        documentId: "twoLevelPartitionKey_empty_null",
+        partitionKeys: [
+          { key: "/twoLevelPartitionKey_1", value: "" },
+          { key: "/twoLevelPartitionKey_2", value: null },
+        ],
+      },
+      {
+        documentId: "twoLevelPartitionKey_null_missing",
+        partitionKeys: [{ key: "/twoLevelPartitionKey_1", value: null }],
+      },
+      {
+        documentId: "twoLevelPartitionKey_missing_value",
+        partitionKeys: [{ key: "/twoLevelPartitionKey_2", value: "value" }],
+      },
+    ],
+  },
+  {
+    name: "2-Level Hierarchical Nested Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "twoLevelNestedPartitionKey",
+    documents: [
+      {
+        documentId: "twoLevelNestedPartitionKey_nested_value_empty",
+        partitionKeys: [
+          { key: "/twoLevelNestedPartitionKey/nested", value: "value" },
+          { key: "/twoLevelNestedPartitionKey/nested_value/nested", value: "" },
+        ],
+      },
+      {
+        documentId: "twoLevelNestedPartitionKey_nested_null_missing",
+        partitionKeys: [{ key: "/twoLevelNestedPartitionKey/nested", value: null }],
+      },
+      {
+        documentId: "twoLevelNestedPartitionKey_nested_missing_value",
+        partitionKeys: [{ key: "/twoLevelNestedPartitionKey/nested_value/nested", value: "value" }],
+      },
+    ],
+  },
+  {
+    name: "3-Level Hierarchical Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "threeLevelPartitionKey",
+    documents: [
+      {
+        documentId: "threeLevelPartitionKey_value_empty_null",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1", value: "value" },
+          { key: "/threeLevelPartitionKey_2", value: "" },
+          { key: "/threeLevelPartitionKey_3", value: null },
+        ],
+      },
+      {
+        documentId: "threeLevelPartitionKey_value_null_missing",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1", value: "value" },
+          { key: "/threeLevelPartitionKey_2", value: null },
+        ],
+      },
+      {
+        documentId: "threeLevelPartitionKey_value_missing_null",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1", value: "value" },
+          { key: "/threeLevelPartitionKey_3", value: null },
+        ],
+      },
+      {
+        documentId: "threeLevelPartitionKey_null_empty_value",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1", value: null },
+          { key: "/threeLevelPartitionKey_2", value: "" },
+          { key: "/threeLevelPartitionKey_3", value: "value" },
+        ],
+      },
+      {
+        documentId: "threeLevelPartitionKey_missing_value_value",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_2", value: "value" },
+          { key: "/threeLevelPartitionKey_3", value: "value" },
+        ],
+      },
+      {
+        documentId: "threeLevelPartitionKey_empty_value_missing",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1", value: "" },
+          { key: "/threeLevelPartitionKey_2", value: "value" },
+        ],
+      },
+    ],
+  },
+  {
+    name: "3-Level Nested Hierarchical Partition Key",
+    databaseId: "e2etests-sql-readonly",
+    containerId: "threeLevelNestedPartitionKey",
+    documents: [
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_empty_value_null",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_key", value: "" },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: "value" },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_3/nested_key", value: null },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_null_value_missing",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_key", value: null },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: "value" },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_missing_value_null",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: "value" },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_3/nested_key", value: null },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_null_empty_missing",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_key", value: null },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: "" },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_value_missing_empty",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_key", value: "value" },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_3/nested_key", value: "" },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_missing_null_empty",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: null },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_3/nested_key", value: "" },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_empty_null_value",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_key", value: "" },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: null },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_3/nested_key", value: "value" },
+        ],
+      },
+      {
+        documentId: "threeLevelNestedPartitionKey_nested_value_null_empty",
+        partitionKeys: [
+          { key: "/threeLevelPartitionKey_1/nested/nested_key", value: "value" },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_key", value: null },
+          { key: "/threeLevelPartitionKey_1/nested/nested_2/nested_3/nested_key", value: "" },
+        ],
+      },
+    ],
+  },
+];

test/testData.ts

@@ -16,6 +16,23 @@ export interface TestItem {
   randomData: string;
 }
 
+export interface DocumentTestCase {
+  name: string;
+  databaseId: string;
+  containerId: string;
+  documents: TestDocument[];
+}
+
+export interface TestDocument {
+  documentId: string;
+  partitionKeys?: PartitionKey[];
+}
+
+export interface PartitionKey {
+  key: string;
+  value: string | null;
+}
+
 const partitionCount = 4;
 
 // If we increase this number, we need to split bulk creates into multiple batches.
@@ -93,3 +110,46 @@ export async function createTestSQLContainer(includeTestData?: boolean) {
     throw e;
   }
 }
+
+export const setPartitionKeys = (partitionKeys: PartitionKey[]) => {
+  const result = {};
+
+  partitionKeys.forEach((partitionKey) => {
+    const { key: keyPath, value: keyValue } = partitionKey;
+    const cleanPath = keyPath.startsWith("/") ? keyPath.slice(1) : keyPath;
+    const keys = cleanPath.split("/");
+    let current = result;
+
+    keys.forEach((key, index) => {
+      if (index === keys.length - 1) {
+        current[key] = keyValue;
+      } else {
+        current[key] = current[key] || {};
+        current = current[key];
+      }
+    });
+  });
+
+  return result;
+};
+
+export const serializeMongoToJson = (text: string) => {
+  const normalized = text.replace(/ObjectId\("([0-9a-fA-F]{24})"\)/g, '"$1"');
+  return JSON.parse(normalized);
+};
+
+export async function retry<T>(fn: () => Promise<T>, retries = 3, delayMs = 1000): Promise<T> {
+  let lastError: unknown;
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      console.warn(`Retry ${i + 1}/${retries} failed: ${(error as Error).message}`);
+      if (i < retries - 1) {
+        await new Promise((res) => setTimeout(res, delayMs));
+      }
+    }
+  }
+  throw lastError;
+}