Better handling throttling error in bulk delete (#1954)

* Implement retry on throttling for nosql

* Clean up code

* Produce specific error for throttling error in mongoProxy bulk delete. Clean up code.

* Fix throttling doc url

* Fix mongo error wording

* Fix unit test

* Unit test cleanup

* Fix format

* Fix unit tests

* Fix format

* Fix unit test

* Fix format

* Improve comments

* Improve error message wording. Fix URL and add specific URL for Mongo and NoSql.

* Fix error messages. Add console errors.

* Clean up selection of various delete fct

* Fix error display

src/Common/Constants.ts

@@ -293,6 +293,7 @@ export class HttpStatusCodes {
   public static readonly Accepted: number = 202;
   public static readonly NoContent: number = 204;
   public static readonly NotModified: number = 304;
+  public static readonly BadRequest: number = 400;
   public static readonly Unauthorized: number = 401;
   public static readonly Forbidden: number = 403;
   public static readonly NotFound: number = 404;

src/Common/MongoProxyClient.ts

@@ -738,6 +738,12 @@ export function useMongoProxyEndpoint(api: string): boolean {
   );
 }
 
+export class ThrottlingError extends Error {
+  constructor(message: string) {
+    super(message);
+  }
+}
+
 // TODO: This function throws most of the time except on Forbidden which is a bit strange
 // It causes problems for TypeScript understanding the types
 async function errorHandling(response: Response, action: string, params: unknown): Promise<void> {
@@ -747,6 +753,14 @@ async function errorHandling(response: Response, action: string, params: unknown
   if (response.status === HttpStatusCodes.Forbidden) {
     sendMessage({ type: MessageTypes.ForbiddenError, reason: errorMessage });
     return;
+  } else if (
+    response.status === HttpStatusCodes.BadRequest &&
+    errorMessage.includes("Error=16500") &&
+    errorMessage.includes("RetryAfterMs=")
+  ) {
+    // If throttling is happening, Cosmos DB will return a 400 with a body of:
+    // A write operation resulted in an error. Error=16500, RetryAfterMs=4, Details='Batch write error.
+    throw new ThrottlingError(errorMessage);
   }
   throw new Error(errorMessage);
 }

src/Common/dataAccess/deleteDocument.ts

@@ -26,14 +26,23 @@ export const deleteDocument = async (collection: CollectionBase, documentId: Doc
   }
 };
 
+export interface IBulkDeleteResult {
+  documentId: DocumentId;
+  requestCharge: number;
+  statusCode: number;
+  retryAfterMilliseconds?: number;
+}
+
 /**
  * Bulk delete documents
  * @param collection
  * @param documentId
- * @returns array of ids that were successfully deleted
+ * @returns array of results and status codes
  */
-export const deleteDocuments = async (collection: CollectionBase, documentIds: DocumentId[]): Promise<DocumentId[]> => {
-  const nbDocuments = documentIds.length;
+export const deleteDocuments = async (
+  collection: CollectionBase,
+  documentIds: DocumentId[],
+): Promise<IBulkDeleteResult[]> => {
   const clearMessage = logConsoleProgress(`Deleting ${documentIds.length} ${getEntityName(true)}`);
   try {
     const v2Container = await client().database(collection.databaseId).container(collection.id());
@@ -56,18 +65,17 @@ export const deleteDocuments = async (collection: CollectionBase, documentIds: D
         operationType: BulkOperationType.Delete,
       }));
 
-      const promise = v2Container.items.bulk(operations).then((bulkResult) => {
-        return documentIdsChunk.filter((_, index) => bulkResult[index].statusCode === 204);
+      const promise = v2Container.items.bulk(operations).then((bulkResults) => {
+        return bulkResults.map((bulkResult, index) => {
+          const documentId = documentIdsChunk[index];
+          return { ...bulkResult, documentId };
+        });
       });
       promiseArray.push(promise);
     }
 
     const allResult = await Promise.all(promiseArray);
     const flatAllResult = Array.prototype.concat.apply([], allResult);
-    logConsoleInfo(
-      `Successfully deleted ${getEntityName(flatAllResult.length > 1)}: ${flatAllResult.length} out of ${nbDocuments}`,
-    );
-    // TODO: handle case result.length != nbDocuments
     return flatAllResult;
   } catch (error) {
     handleError(