Merge branch 'master' into users/languy/redact-sensitive-info-from-telemetry

src/Common/ErrorHandlingUtils.ts

@@ -7,16 +7,27 @@ import { HttpStatusCodes } from "./Constants";
 import { logError } from "./Logger";
 import { sendMessage } from "./MessageHandler";
 
-export const handleError = (error: string | ARMError | Error, area: string, consoleErrorPrefix?: string): void => {
+export interface HandleErrorOptions {
+  /** Optional redacted error to use for telemetry logging instead of the original error */
+  redactedError?: string | ARMError | Error;
+}
+
+export const handleError = (
+  error: string | ARMError | Error,
+  area: string,
+  consoleErrorPrefix?: string,
+  options?: HandleErrorOptions,
+): void => {
   const errorMessage = getErrorMessage(error);
   const errorCode = error instanceof ARMError ? error.code : undefined;
 
-  // logs error to data explorer console
+  // logs error to data explorer console (always shows original, non-redacted message)
   const consoleErrorMessage = consoleErrorPrefix ? `${consoleErrorPrefix}:\n ${errorMessage}` : errorMessage;
   logConsoleError(consoleErrorMessage);
 
-  // logs error to both app insight and kusto
-  logError(errorMessage, area, errorCode);
+  // logs error to both app insight and kusto (use redacted message if provided)
+  const telemetryErrorMessage = options?.redactedError ? getErrorMessage(options.redactedError) : errorMessage;
+  logError(telemetryErrorMessage, area, errorCode);
 
   // checks for errors caused by firewall and sends them to portal to handle
   sendNotificationForError(errorMessage, errorCode);

src/Common/MongoProxyClient.ts

@@ -38,7 +38,7 @@ export function queryIterator(databaseId: string, collection: Collection, query:
   let continuationToken: string;
   return {
     fetchNext: () => {
-      return queryDocuments(databaseId, collection, false, query, continuationToken).then((response) => {
+      return queryDocuments(databaseId, collection, false, query).then((response) => {
         continuationToken = response.continuationToken;
         const headers: { [key: string]: string | number } = {};
         response.headers.forEach((value, key) => {

src/Common/dataAccess/queryDocumentsPage.test.ts

@@ -0,0 +1,171 @@
+import { redactSyntaxErrorMessage } from "./queryDocumentsPage";
+
+/* Typical error to redact looks like this (the message property contains a JSON string with nested structure):
+{
+  "message": "{\"code\":\"BadRequest\",\"message\":\"{\\\"errors\\\":[{\\\"severity\\\":\\\"Error\\\",\\\"location\\\":{\\\"start\\\":0,\\\"end\\\":5},\\\"code\\\":\\\"SC1001\\\",\\\"message\\\":\\\"Syntax error, incorrect syntax near 'Crazy'.\\\"}]}\\r\\nActivityId: d5424e10-51bd-46f7-9aec-7b40bed36f17, Windows/10.0.20348 cosmos-netstandard-sdk/3.18.0\"}"
+}
+*/
+
+// Helper to create the nested error structure that matches what the SDK returns
+const createNestedError = (
+  errors: Array<{ severity?: string; location?: { start: number; end: number }; code: string; message: string }>,
+  activityId: string = "test-activity-id",
+): { message: string } => {
+  const innerErrorsJson = JSON.stringify({ errors });
+  const innerMessage = `${innerErrorsJson}\r\n${activityId}`;
+  const outerJson = JSON.stringify({ code: "BadRequest", message: innerMessage });
+  return { message: outerJson };
+};
+
+// Helper to parse the redacted result
+const parseRedactedResult = (result: { message: string }) => {
+  const outerParsed = JSON.parse(result.message);
+  const [innerErrorsJson, activityIdPart] = outerParsed.message.split("\r\n");
+  const innerErrors = JSON.parse(innerErrorsJson);
+  return { outerParsed, innerErrors, activityIdPart };
+};
+
+describe("redactSyntaxErrorMessage", () => {
+  it("should redact SC1001 error message", () => {
+    const error = createNestedError(
+      [
+        {
+          severity: "Error",
+          location: { start: 0, end: 5 },
+          code: "SC1001",
+          message: "Syntax error, incorrect syntax near 'Crazy'.",
+        },
+      ],
+      "ActivityId: d5424e10-51bd-46f7-9aec-7b40bed36f17",
+    );
+
+    const result = redactSyntaxErrorMessage(error) as { message: string };
+    const { outerParsed, innerErrors, activityIdPart } = parseRedactedResult(result);
+
+    expect(outerParsed.code).toBe("BadRequest");
+    expect(innerErrors.errors[0].message).toBe("__REDACTED__");
+    expect(activityIdPart).toContain("ActivityId: d5424e10-51bd-46f7-9aec-7b40bed36f17");
+  });
+
+  it("should redact SC2001 error message", () => {
+    const error = createNestedError(
+      [
+        {
+          severity: "Error",
+          location: { start: 0, end: 10 },
+          code: "SC2001",
+          message: "Some sensitive syntax error message.",
+        },
+      ],
+      "ActivityId: abc123",
+    );
+
+    const result = redactSyntaxErrorMessage(error) as { message: string };
+    const { outerParsed, innerErrors, activityIdPart } = parseRedactedResult(result);
+
+    expect(outerParsed.code).toBe("BadRequest");
+    expect(innerErrors.errors[0].message).toBe("__REDACTED__");
+    expect(activityIdPart).toContain("ActivityId: abc123");
+  });
+
+  it("should redact multiple errors with SC1001 and SC2001 codes", () => {
+    const error = createNestedError(
+      [
+        { severity: "Error", code: "SC1001", message: "First error" },
+        { severity: "Error", code: "SC2001", message: "Second error" },
+      ],
+      "ActivityId: xyz",
+    );
+
+    const result = redactSyntaxErrorMessage(error) as { message: string };
+    const { innerErrors } = parseRedactedResult(result);
+
+    expect(innerErrors.errors[0].message).toBe("__REDACTED__");
+    expect(innerErrors.errors[1].message).toBe("__REDACTED__");
+  });
+
+  it("should not redact errors with other codes", () => {
+    const error = createNestedError(
+      [{ severity: "Error", code: "SC9999", message: "This should not be redacted." }],
+      "ActivityId: test123",
+    );
+
+    const result = redactSyntaxErrorMessage(error);
+
+    expect(result).toBe(error); // Should return original error unchanged
+  });
+
+  it("should not modify non-BadRequest errors", () => {
+    const innerMessage = JSON.stringify({ errors: [{ code: "SC1001", message: "Should not be redacted" }] });
+    const error = {
+      message: JSON.stringify({ code: "NotFound", message: innerMessage }),
+    };
+
+    const result = redactSyntaxErrorMessage(error);
+
+    expect(result).toBe(error);
+  });
+
+  it("should handle errors without message property", () => {
+    const error = { code: "BadRequest" };
+
+    const result = redactSyntaxErrorMessage(error);
+
+    expect(result).toBe(error);
+  });
+
+  it("should handle non-object errors", () => {
+    const stringError = "Simple string error";
+    const nullError: null = null;
+    const undefinedError: undefined = undefined;
+
+    expect(redactSyntaxErrorMessage(stringError)).toBe(stringError);
+    expect(redactSyntaxErrorMessage(nullError)).toBe(nullError);
+    expect(redactSyntaxErrorMessage(undefinedError)).toBe(undefinedError);
+  });
+
+  it("should handle malformed JSON in message", () => {
+    const error = {
+      message: "not valid json",
+    };
+
+    const result = redactSyntaxErrorMessage(error);
+
+    expect(result).toBe(error);
+  });
+
+  it("should handle message without ActivityId suffix", () => {
+    const innerErrorsJson = JSON.stringify({
+      errors: [{ severity: "Error", code: "SC1001", message: "Syntax error near something." }],
+    });
+    const error = {
+      message: JSON.stringify({ code: "BadRequest", message: innerErrorsJson + "\r\n" }),
+    };
+
+    const result = redactSyntaxErrorMessage(error) as { message: string };
+    const { innerErrors } = parseRedactedResult(result);
+
+    expect(innerErrors.errors[0].message).toBe("__REDACTED__");
+  });
+
+  it("should preserve other error properties", () => {
+    const baseError = createNestedError([{ code: "SC1001", message: "Error" }], "ActivityId: test");
+    const error = {
+      ...baseError,
+      statusCode: 400,
+      additionalInfo: "extra data",
+    };
+
+    const result = redactSyntaxErrorMessage(error) as {
+      message: string;
+      statusCode: number;
+      additionalInfo: string;
+    };
+
+    expect(result.statusCode).toBe(400);
+    expect(result.additionalInfo).toBe("extra data");
+
+    const { innerErrors } = parseRedactedResult(result);
+    expect(innerErrors.errors[0].message).toBe("__REDACTED__");
+  });
+});

src/Common/dataAccess/queryDocumentsPage.ts

@@ -4,6 +4,51 @@ import { getEntityName } from "../DocumentUtility";
 import { handleError } from "../ErrorHandlingUtils";
 import { MinimalQueryIterator, nextPage } from "../IteratorUtilities";
 
+// Redact sensitive information from BadRequest errors with specific codes
+export const redactSyntaxErrorMessage = (error: unknown): unknown => {
+  const codesToRedact = ["SC1001", "SC2001"];
+
+  try {
+    // Handle error objects with a message property
+    if (error && typeof error === "object" && "message" in error) {
+      const errorObj = error as { code?: string; message?: string };
+      if (typeof errorObj.message === "string") {
+        // Parse the inner JSON from the message
+        const innerJson = JSON.parse(errorObj.message);
+        if (innerJson.code === "BadRequest" && typeof innerJson.message === "string") {
+          const [innerErrorsJson, activityIdPart] = innerJson.message.split("\r\n");
+          const innerErrorsObj = JSON.parse(innerErrorsJson);
+          if (Array.isArray(innerErrorsObj.errors)) {
+            let modified = false;
+            innerErrorsObj.errors = innerErrorsObj.errors.map((err: { code?: string; message?: string }) => {
+              if (err.code && codesToRedact.includes(err.code)) {
+                modified = true;
+                return { ...err, message: "__REDACTED__" };
+              }
+              return err;
+            });
+
+            if (modified) {
+              // Reconstruct the message with the redacted content
+              const redactedMessage = JSON.stringify(innerErrorsObj) + `\r\n${activityIdPart}`;
+              const redactedError = {
+                ...error,
+                message: JSON.stringify({ ...innerJson, message: redactedMessage }),
+                body: undefined as unknown, // Clear body to avoid sensitive data
+              };
+              return redactedError;
+            }
+          }
+        }
+      }
+    }
+  } catch {
+    // If parsing fails, return the original error
+  }
+
+  return error;
+};
+
 export const queryDocumentsPage = async (
   resourceName: string,
   documentsIterator: MinimalQueryIterator,
@@ -18,7 +63,12 @@ export const queryDocumentsPage = async (
     logConsoleInfo(`Successfully fetched ${itemCount} ${entityName} for container ${resourceName}`);
     return result;
   } catch (error) {
-    handleError(error, "QueryDocumentsPage", `Failed to query ${entityName} for container ${resourceName}`);
+    // Redact sensitive information for telemetry while showing original in console
+    const redactedError = redactSyntaxErrorMessage(error);
+
+    handleError(error, "QueryDocumentsPage", `Failed to query ${entityName} for container ${resourceName}`, {
+      redactedError: redactedError as Error,
+    });
     throw error;
   } finally {
     clearMessage();

test/mongo/pagination.spec.ts

@@ -1,132 +0,0 @@
-import { expect, test } from "@playwright/test";
-import { setupCORSBypass } from "../CORSBypass";
-import { DataExplorer, QueryTab, TestAccount, CommandBarButton, Editor } from "../fx";
-import { serializeMongoToJson } from "../testData";
-
-const databaseId = "test-e2etests-mongo-pagination";
-const collectionId = "test-coll-mongo-pagination";
-let explorer: DataExplorer = null!;
-
-test.setTimeout(5 * 60 * 1000);
-
-test.describe("Test Mongo Pagination", () => {
-  let queryTab: QueryTab;
-  let queryEditor: Editor;
-
-  test.beforeEach("Open query tab", async ({ page }) => {
-    await setupCORSBypass(page);
-    explorer = await DataExplorer.open(page, TestAccount.MongoReadonly);
-
-    const containerNode = await explorer.waitForContainerNode(databaseId, collectionId);
-    await containerNode.expand();
-
-    const containerMenuNode = await explorer.waitForContainerDocumentsNode(databaseId, collectionId);
-    await containerMenuNode.openContextMenu();
-    await containerMenuNode.contextMenuItem("New Query").click();
-
-    queryTab = explorer.queryTab("tab0");
-    queryEditor = queryTab.editor();
-    await queryEditor.locator.waitFor({ timeout: 30 * 1000 });
-    await queryTab.executeCTA.waitFor();
-    await explorer.frame.getByTestId("NotificationConsole/ExpandCollapseButton").click();
-    await explorer.frame.getByTestId("NotificationConsole/Contents").waitFor();
-  });
-
-  test("should execute a query and load more results", async ({ page }) => {
-    const query = "{}";
-
-    await queryEditor.locator.click();
-    await queryEditor.setText(query);
-
-    const executeQueryButton = explorer.commandBarButton(CommandBarButton.ExecuteQuery);
-    await executeQueryButton.click();
-
-    // Wait for query execution to complete
-    await expect(queryTab.resultsView).toBeVisible({ timeout: 60000 });
-    await expect(queryTab.resultsEditor.locator).toBeAttached({ timeout: 30000 });
-
-    // Get initial results
-    const resultText = await queryTab.resultsEditor.text();
-
-    if (!resultText || resultText.trim() === "" || resultText.trim() === "[]") {
-      throw new Error("Query returned no results - the collection appears to be empty");
-    }
-
-    const resultData = serializeMongoToJson(resultText);
-
-    if (resultData.length === 0) {
-      throw new Error("Parsed results contain 0 documents - collection is empty");
-    }
-
-    if (resultData.length < 100) {
-      expect(resultData.length).toBeGreaterThan(0);
-      return;
-    }
-
-    expect(resultData.length).toBe(100);
-
-    // Pagination test
-    let totalPagesLoaded = 1;
-    const maxLoadMoreAttempts = 10;
-
-    for (let loadMoreAttempts = 0; loadMoreAttempts < maxLoadMoreAttempts; loadMoreAttempts++) {
-      const loadMoreButton = queryTab.resultsView.getByText("Load more");
-
-      try {
-        await expect(loadMoreButton).toBeVisible({ timeout: 5000 });
-      } catch {
-        // Load more button not visible - pagination complete
-        break;
-      }
-
-      const beforeClickText = await queryTab.resultsEditor.text();
-      const beforeClickHash = Buffer.from(beforeClickText || "")
-        .toString("base64")
-        .substring(0, 50);
-
-      await loadMoreButton.click();
-
-      // Wait for content to update
-      let editorContentChanged = false;
-      for (let waitAttempt = 1; waitAttempt <= 3; waitAttempt++) {
-        await page.waitForTimeout(2000);
-
-        const currentEditorText = await queryTab.resultsEditor.text();
-        const currentHash = Buffer.from(currentEditorText || "")
-          .toString("base64")
-          .substring(0, 50);
-
-        if (currentHash !== beforeClickHash) {
-          editorContentChanged = true;
-          break;
-        }
-      }
-
-      if (editorContentChanged) {
-        totalPagesLoaded++;
-      } else {
-        // No content change detected, stop pagination
-        break;
-      }
-
-      await page.waitForTimeout(1000);
-    }
-
-    // Final verification
-    const finalIndicator = queryTab.resultsView.locator("text=/\\d+ - \\d+/");
-    const finalIndicatorText = await finalIndicator.textContent();
-
-    if (finalIndicatorText) {
-      const match = finalIndicatorText.match(/(\d+) - (\d+)/);
-      if (match) {
-        const totalDocuments = parseInt(match[2]);
-        expect(totalDocuments).toBe(405);
-        expect(totalPagesLoaded).toBe(5);
-      } else {
-        throw new Error(`Invalid results indicator format: ${finalIndicatorText}`);
-      }
-    } else {
-      expect(totalPagesLoaded).toBe(5);
-    }
-  });
-});