Query Max Retry settings (#1688)

* Added query retry settings

* prettier run

* Fixed tests and queryDocuments

* Fixed tests

* corrected logic

* Updated tests and logic

* Removed optional flag

* Added default value text

* Reworded text

* moved retry options to CosmosClient

* removed unused references to retryOptions

* Reverting formatting

* reverting

* revert

* prettier run

* Correct default and added options directly to the client

* Prettier run and unit test update

* Corrected tooltip and constant name

* Added inSeconds to WaitTime

src/Common/Constants.ts

@@ -145,6 +145,9 @@ export class Queries {
   public static QueryEditorMinHeightRatio: number = 0.1;
   public static QueryEditorMaxHeightRatio: number = 0.4;
   public static readonly DefaultMaxDegreeOfParallelism = 6;
+  public static readonly DefaultRetryAttempts = 9;
+  public static readonly DefaultRetryIntervalInMs = 0;
+  public static readonly DefaultMaxWaitTimeInSeconds = 30;
 }
 
 export class SavedQueries {

src/Common/CosmosClient.ts

@@ -3,6 +3,7 @@ import { sendCachedDataMessage } from "Common/MessageHandler";
 import { getAuthorizationTokenUsingResourceTokens } from "Common/getAuthorizationTokenUsingResourceTokens";
 import { AuthorizationToken, MessageTypes } from "Contracts/MessageTypes";
 import { checkDatabaseResourceTokensValidity } from "Platform/Fabric/FabricUtil";
+import { LocalStorageUtility, StorageKey } from "Shared/StorageUtility";
 import { AuthType } from "../AuthType";
 import { PriorityLevel } from "../Common/Constants";
 import { Platform, configContext } from "../ConfigContext";
@@ -150,6 +151,13 @@ export function client(): Cosmos.CosmosClient {
     tokenProvider,
     userAgentSuffix: "Azure Portal",
     defaultHeaders: _defaultHeaders,
+    connectionPolicy: {
+      retryOptions: {
+        maxRetryAttemptCount: LocalStorageUtility.getEntryNumber(StorageKey.RetryAttempts),
+        fixedRetryIntervalInMilliseconds: LocalStorageUtility.getEntryNumber(StorageKey.RetryInterval),
+        maxWaitTimeInSeconds: LocalStorageUtility.getEntryNumber(StorageKey.MaxWaitTimeInSeconds),
+      },
+    },
   };
 
   if (configContext.PROXY_PATH !== undefined) {

src/Common/QueriesClient.ts

@@ -1,3 +1,4 @@
+import { isServerlessAccount } from "Utils/CapabilityUtils";
 import * as _ from "underscore";
 import * as DataModels from "../Contracts/DataModels";
 import * as ViewModels from "../Contracts/ViewModels";
@@ -8,12 +9,11 @@ import { useDatabases } from "../Explorer/useDatabases";
 import { userContext } from "../UserContext";
 import * as NotificationConsoleUtils from "../Utils/NotificationConsoleUtils";
 import { BackendDefaults, HttpStatusCodes, SavedQueries } from "./Constants";
+import { handleError } from "./ErrorHandlingUtils";
 import { createCollection } from "./dataAccess/createCollection";
 import { createDocument } from "./dataAccess/createDocument";
 import { deleteDocument } from "./dataAccess/deleteDocument";
 import { queryDocuments } from "./dataAccess/queryDocuments";
-import { handleError } from "./ErrorHandlingUtils";
-import { isServerlessAccount } from "Utils/CapabilityUtils";
 
 export class QueriesClient {
   private static readonly PartitionKey: DataModels.PartitionKey = {

src/Common/dataAccess/queryDocuments.test.ts

@@ -1,5 +1,5 @@
-import { getCommonQueryOptions } from "./queryDocuments";
 import { LocalStorageUtility, StorageKey } from "../../Shared/StorageUtility";
+import { getCommonQueryOptions } from "./queryDocuments";
 
 describe("getCommonQueryOptions", () => {
   it("builds the correct default options objects", () => {

src/Common/dataAccess/queryDocuments.ts

@@ -26,6 +26,5 @@ export const getCommonQueryOptions = (options: FeedOptions): FeedOptions => {
     (storedItemPerPageSetting !== undefined && storedItemPerPageSetting) ||
     Queries.itemsPerPage;
   options.maxDegreeOfParallelism = LocalStorageUtility.getEntryNumber(StorageKey.MaxDegreeOfParellism);
-
   return options;
 };

src/Explorer/Panes/SettingsPane/SettingsPane.tsx

@@ -53,6 +53,21 @@ export const SettingsPane: FunctionComponent = () => {
       ? LocalStorageUtility.getEntryString(StorageKey.IsGraphAutoVizDisabled)
       : "false",
   );
+  const [retryAttempts, setRetryAttempts] = useState<number>(
+    LocalStorageUtility.hasItem(StorageKey.RetryAttempts)
+      ? LocalStorageUtility.getEntryNumber(StorageKey.RetryAttempts)
+      : Constants.Queries.DefaultRetryAttempts,
+  );
+  const [retryInterval, setRetryInterval] = useState<number>(
+    LocalStorageUtility.hasItem(StorageKey.RetryInterval)
+      ? LocalStorageUtility.getEntryNumber(StorageKey.RetryInterval)
+      : Constants.Queries.DefaultRetryIntervalInMs,
+  );
+  const [MaxWaitTimeInSeconds, setMaxWaitTimeInSeconds] = useState<number>(
+    LocalStorageUtility.hasItem(StorageKey.MaxWaitTimeInSeconds)
+      ? LocalStorageUtility.getEntryNumber(StorageKey.MaxWaitTimeInSeconds)
+      : Constants.Queries.DefaultMaxWaitTimeInSeconds,
+  );
   const [maxDegreeOfParallelism, setMaxDegreeOfParallelism] = useState<number>(
     LocalStorageUtility.hasItem(StorageKey.MaxDegreeOfParellism)
       ? LocalStorageUtility.getEntryNumber(StorageKey.MaxDegreeOfParellism)
@@ -78,6 +93,9 @@ export const SettingsPane: FunctionComponent = () => {
     );
     LocalStorageUtility.setEntryNumber(StorageKey.CustomItemPerPage, customItemPerPage);
     LocalStorageUtility.setEntryBoolean(StorageKey.QueryTimeoutEnabled, queryTimeoutEnabled);
+    LocalStorageUtility.setEntryNumber(StorageKey.RetryAttempts, retryAttempts);
+    LocalStorageUtility.setEntryNumber(StorageKey.RetryInterval, retryInterval);
+    LocalStorageUtility.setEntryNumber(StorageKey.MaxWaitTimeInSeconds, MaxWaitTimeInSeconds);
     LocalStorageUtility.setEntryString(StorageKey.ContainerPaginationEnabled, containerPaginationEnabled.toString());
     LocalStorageUtility.setEntryString(StorageKey.IsCrossPartitionQueryEnabled, crossPartitionQueryEnabled.toString());
     LocalStorageUtility.setEntryNumber(StorageKey.MaxDegreeOfParellism, maxDegreeOfParallelism);
@@ -179,6 +197,27 @@ export const SettingsPane: FunctionComponent = () => {
     }
   };
 
+  const handleOnQueryRetryAttemptsSpinButtonChange = (ev: React.MouseEvent<HTMLElement>, newValue?: string): void => {
+    const retryAttempts = Number(newValue);
+    if (!isNaN(retryAttempts)) {
+      setRetryAttempts(retryAttempts);
+    }
+  };
+
+  const handleOnRetryIntervalSpinButtonChange = (ev: React.MouseEvent<HTMLElement>, newValue?: string): void => {
+    const retryInterval = Number(newValue);
+    if (!isNaN(retryInterval)) {
+      setRetryInterval(retryInterval);
+    }
+  };
+
+  const handleOnMaxWaitTimeSpinButtonChange = (ev: React.MouseEvent<HTMLElement>, newValue?: string): void => {
+    const MaxWaitTimeInSeconds = Number(newValue);
+    if (!isNaN(MaxWaitTimeInSeconds)) {
+      setMaxWaitTimeInSeconds(MaxWaitTimeInSeconds);
+    }
+  };
+
   const choiceButtonStyles = {
     root: {
       clear: "both",
@@ -323,6 +362,77 @@ export const SettingsPane: FunctionComponent = () => {
             </div>
           </div>
         )}
+        <div className="settingsSection">
+          <div className="settingsSectionPart">
+            <div className="settingsSectionLabel">
+              Retry Settings
+              <InfoTooltip>Retry policy associated with throttled requests during CosmosDB queries.</InfoTooltip>
+            </div>
+            <div>
+              <legend id="queryRetryAttemptsLabel" className="settingsSectionLabel legendLabel">
+                Max retry attempts
+              </legend>
+              <InfoTooltip>Max number of retries to be performed for a request. Default value 9.</InfoTooltip>
+            </div>
+            <SpinButton
+              labelPosition={Position.top}
+              min={1}
+              step={1}
+              value={"" + retryAttempts}
+              onChange={handleOnQueryRetryAttemptsSpinButtonChange}
+              incrementButtonAriaLabel="Increase value by 1"
+              decrementButtonAriaLabel="Decrease value by 1"
+              onIncrement={(newValue) => setRetryAttempts(parseInt(newValue) + 1 || retryAttempts)}
+              onDecrement={(newValue) => setRetryAttempts(parseInt(newValue) - 1 || retryAttempts)}
+              onValidate={(newValue) => setRetryAttempts(parseInt(newValue) || retryAttempts)}
+              styles={queryTimeoutSpinButtonStyles}
+            />
+            <div>
+              <legend id="queryRetryAttemptsLabel" className="settingsSectionLabel legendLabel">
+                Fixed retry interval (ms)
+              </legend>
+              <InfoTooltip>
+                Fixed retry interval in milliseconds to wait between each retry ignoring the retryAfter returned as part
+                of the response. Default value is 0 milliseconds.
+              </InfoTooltip>
+            </div>
+            <SpinButton
+              labelPosition={Position.top}
+              min={1000}
+              step={1000}
+              value={"" + retryInterval}
+              onChange={handleOnRetryIntervalSpinButtonChange}
+              incrementButtonAriaLabel="Increase value by 1000"
+              decrementButtonAriaLabel="Decrease value by 1000"
+              onIncrement={(newValue) => setRetryInterval(parseInt(newValue) + 1000 || retryInterval)}
+              onDecrement={(newValue) => setRetryInterval(parseInt(newValue) - 1000 || retryInterval)}
+              onValidate={(newValue) => setRetryInterval(parseInt(newValue) || retryInterval)}
+              styles={queryTimeoutSpinButtonStyles}
+            />
+            <div>
+              <legend id="queryRetryAttemptsLabel" className="settingsSectionLabel legendLabel">
+                Max wait time (s)
+              </legend>
+              <InfoTooltip>
+                Max wait time in seconds to wait for a request while the retries are happening. Default value 30
+                seconds.
+              </InfoTooltip>
+            </div>
+            <SpinButton
+              labelPosition={Position.top}
+              min={1}
+              step={1}
+              value={"" + MaxWaitTimeInSeconds}
+              onChange={handleOnMaxWaitTimeSpinButtonChange}
+              incrementButtonAriaLabel="Increase value by 1"
+              decrementButtonAriaLabel="Decrease value by 1"
+              onIncrement={(newValue) => setMaxWaitTimeInSeconds(parseInt(newValue) + 1 || MaxWaitTimeInSeconds)}
+              onDecrement={(newValue) => setMaxWaitTimeInSeconds(parseInt(newValue) - 1 || MaxWaitTimeInSeconds)}
+              onValidate={(newValue) => setMaxWaitTimeInSeconds(parseInt(newValue) || MaxWaitTimeInSeconds)}
+              styles={queryTimeoutSpinButtonStyles}
+            />
+          </div>
+        </div>
         <div className="settingsSection">
           <div className="settingsSectionPart">
             <div className="settingsSectionLabel">

src/Explorer/Panes/SettingsPane/__snapshots__/SettingsPane.test.tsx.snap

@@ -137,6 +137,139 @@ exports[`Settings Pane should render Default properly 1`] = `
         </div>
       </div>
     </div>
+    <div
+      className="settingsSection"
+    >
+      <div
+        className="settingsSectionPart"
+      >
+        <div
+          className="settingsSectionLabel"
+        >
+          Retry Settings
+          <InfoTooltip>
+            Retry policy associated with throttled requests during CosmosDB queries.
+          </InfoTooltip>
+        </div>
+        <div>
+          <legend
+            className="settingsSectionLabel legendLabel"
+            id="queryRetryAttemptsLabel"
+          >
+            Max retry attempts
+          </legend>
+          <InfoTooltip>
+            Max number of retries to be performed for a request. Default value 9.
+          </InfoTooltip>
+        </div>
+        <StyledSpinButton
+          decrementButtonAriaLabel="Decrease value by 1"
+          incrementButtonAriaLabel="Increase value by 1"
+          labelPosition={0}
+          min={1}
+          onChange={[Function]}
+          onDecrement={[Function]}
+          onIncrement={[Function]}
+          onValidate={[Function]}
+          step={1}
+          styles={
+            Object {
+              "arrowButtonsContainer": Object {},
+              "icon": Object {},
+              "input": Object {},
+              "label": Object {
+                "fontSize": 12,
+                "fontWeight": 400,
+              },
+              "labelWrapper": Object {},
+              "root": Object {
+                "paddingBottom": 10,
+              },
+              "spinButtonWrapper": Object {},
+            }
+          }
+          value="9"
+        />
+        <div>
+          <legend
+            className="settingsSectionLabel legendLabel"
+            id="queryRetryAttemptsLabel"
+          >
+            Fixed retry interval (ms)
+          </legend>
+          <InfoTooltip>
+            Fixed retry interval in milliseconds to wait between each retry ignoring the retryAfter returned as part of the response. Default value is 0 milliseconds.
+          </InfoTooltip>
+        </div>
+        <StyledSpinButton
+          decrementButtonAriaLabel="Decrease value by 1000"
+          incrementButtonAriaLabel="Increase value by 1000"
+          labelPosition={0}
+          min={1000}
+          onChange={[Function]}
+          onDecrement={[Function]}
+          onIncrement={[Function]}
+          onValidate={[Function]}
+          step={1000}
+          styles={
+            Object {
+              "arrowButtonsContainer": Object {},
+              "icon": Object {},
+              "input": Object {},
+              "label": Object {
+                "fontSize": 12,
+                "fontWeight": 400,
+              },
+              "labelWrapper": Object {},
+              "root": Object {
+                "paddingBottom": 10,
+              },
+              "spinButtonWrapper": Object {},
+            }
+          }
+          value="0"
+        />
+        <div>
+          <legend
+            className="settingsSectionLabel legendLabel"
+            id="queryRetryAttemptsLabel"
+          >
+            Max wait time (s)
+          </legend>
+          <InfoTooltip>
+            Max wait time in seconds to wait for a request while the retries are happening. Default value 30 seconds.
+          </InfoTooltip>
+        </div>
+        <StyledSpinButton
+          decrementButtonAriaLabel="Decrease value by 1"
+          incrementButtonAriaLabel="Increase value by 1"
+          labelPosition={0}
+          min={1}
+          onChange={[Function]}
+          onDecrement={[Function]}
+          onIncrement={[Function]}
+          onValidate={[Function]}
+          step={1}
+          styles={
+            Object {
+              "arrowButtonsContainer": Object {},
+              "icon": Object {},
+              "input": Object {},
+              "label": Object {
+                "fontSize": 12,
+                "fontWeight": 400,
+              },
+              "labelWrapper": Object {},
+              "root": Object {
+                "paddingBottom": 10,
+              },
+              "spinButtonWrapper": Object {},
+            }
+          }
+          value="30"
+        />
+      </div>
+    </div>
     <div
       className="settingsSection"
     >
@@ -251,6 +384,139 @@ exports[`Settings Pane should render Gremlin properly 1`] = `
   <div
     className="paneMainContent"
   >
+    <div
+      className="settingsSection"
+    >
+      <div
+        className="settingsSectionPart"
+      >
+        <div
+          className="settingsSectionLabel"
+        >
+          Retry Settings
+          <InfoTooltip>
+            Retry policy associated with throttled requests during CosmosDB queries.
+          </InfoTooltip>
+        </div>
+        <div>
+          <legend
+            className="settingsSectionLabel legendLabel"
+            id="queryRetryAttemptsLabel"
+          >
+            Max retry attempts
+          </legend>
+          <InfoTooltip>
+            Max number of retries to be performed for a request. Default value 9.
+          </InfoTooltip>
+        </div>
+        <StyledSpinButton
+          decrementButtonAriaLabel="Decrease value by 1"
+          incrementButtonAriaLabel="Increase value by 1"
+          labelPosition={0}
+          min={1}
+          onChange={[Function]}
+          onDecrement={[Function]}
+          onIncrement={[Function]}
+          onValidate={[Function]}
+          step={1}
+          styles={
+            Object {
+              "arrowButtonsContainer": Object {},
+              "icon": Object {},
+              "input": Object {},
+              "label": Object {
+                "fontSize": 12,
+                "fontWeight": 400,
+              },
+              "labelWrapper": Object {},
+              "root": Object {
+                "paddingBottom": 10,
+              },
+              "spinButtonWrapper": Object {},
+            }
+          }
+          value="9"
+        />
+        <div>
+          <legend
+            className="settingsSectionLabel legendLabel"
+            id="queryRetryAttemptsLabel"
+          >
+            Fixed retry interval (ms)
+          </legend>
+          <InfoTooltip>
+            Fixed retry interval in milliseconds to wait between each retry ignoring the retryAfter returned as part of the response. Default value is 0 milliseconds.
+          </InfoTooltip>
+        </div>
+        <StyledSpinButton
+          decrementButtonAriaLabel="Decrease value by 1000"
+          incrementButtonAriaLabel="Increase value by 1000"
+          labelPosition={0}
+          min={1000}
+          onChange={[Function]}
+          onDecrement={[Function]}
+          onIncrement={[Function]}
+          onValidate={[Function]}
+          step={1000}
+          styles={
+            Object {
+              "arrowButtonsContainer": Object {},
+              "icon": Object {},
+              "input": Object {},
+              "label": Object {
+                "fontSize": 12,
+                "fontWeight": 400,
+              },
+              "labelWrapper": Object {},
+              "root": Object {
+                "paddingBottom": 10,
+              },
+              "spinButtonWrapper": Object {},
+            }
+          }
+          value="0"
+        />
+        <div>
+          <legend
+            className="settingsSectionLabel legendLabel"
+            id="queryRetryAttemptsLabel"
+          >
+            Max wait time (s)
+          </legend>
+          <InfoTooltip>
+            Max wait time in seconds to wait for a request while the retries are happening. Default value 30 seconds.
+          </InfoTooltip>
+        </div>
+        <StyledSpinButton
+          decrementButtonAriaLabel="Decrease value by 1"
+          incrementButtonAriaLabel="Increase value by 1"
+          labelPosition={0}
+          min={1}
+          onChange={[Function]}
+          onDecrement={[Function]}
+          onIncrement={[Function]}
+          onValidate={[Function]}
+          step={1}
+          styles={
+            Object {
+              "arrowButtonsContainer": Object {},
+              "icon": Object {},
+              "input": Object {},
+              "label": Object {
+                "fontSize": 12,
+                "fontWeight": 400,
+              },
+              "labelWrapper": Object {},
+              "root": Object {
+                "paddingBottom": 10,
+              },
+              "spinButtonWrapper": Object {},
+            }
+          }
+          value="30"
+        />
+      </div>
+    </div>
     <div
       className="settingsSection"
     >

src/Explorer/Tables/TableDataClient.ts

@@ -3,12 +3,12 @@ import * as ko from "knockout";
 import Q from "q";
 import { AuthType } from "../../AuthType";
 import * as Constants from "../../Common/Constants";
+import { handleError } from "../../Common/ErrorHandlingUtils";
+import * as HeadersUtility from "../../Common/HeadersUtility";
 import { createDocument } from "../../Common/dataAccess/createDocument";
 import { deleteDocument } from "../../Common/dataAccess/deleteDocument";
 import { queryDocuments } from "../../Common/dataAccess/queryDocuments";
 import { updateDocument } from "../../Common/dataAccess/updateDocument";
-import { handleError } from "../../Common/ErrorHandlingUtils";
-import * as HeadersUtility from "../../Common/HeadersUtility";
 import { configContext } from "../../ConfigContext";
 import * as ViewModels from "../../Contracts/ViewModels";
 import { userContext } from "../../UserContext";

src/Shared/ExplorerSettings.ts

@@ -6,6 +6,9 @@ export const createDefaultSettings = () => {
   LocalStorageUtility.setEntryNumber(StorageKey.CustomItemPerPage, Constants.Queries.itemsPerPage);
   LocalStorageUtility.setEntryString(StorageKey.IsCrossPartitionQueryEnabled, "true");
   LocalStorageUtility.setEntryNumber(StorageKey.MaxDegreeOfParellism, Constants.Queries.DefaultMaxDegreeOfParallelism);
+  LocalStorageUtility.setEntryNumber(StorageKey.RetryAttempts, Constants.Queries.DefaultRetryAttempts);
+  LocalStorageUtility.setEntryNumber(StorageKey.RetryInterval, Constants.Queries.DefaultRetryIntervalInMs);
+  LocalStorageUtility.setEntryNumber(StorageKey.MaxWaitTimeInSeconds, Constants.Queries.DefaultMaxWaitTimeInSeconds);
   LocalStorageUtility.setEntryString(StorageKey.PriorityLevel, Constants.PriorityLevel.Default);
 };
 
@@ -13,7 +16,10 @@ export const hasSettingsDefined = (): boolean => {
   return (
     LocalStorageUtility.hasItem(StorageKey.ActualItemPerPage) &&
     LocalStorageUtility.hasItem(StorageKey.IsCrossPartitionQueryEnabled) &&
-    LocalStorageUtility.hasItem(StorageKey.MaxDegreeOfParellism)
+    LocalStorageUtility.hasItem(StorageKey.MaxDegreeOfParellism) &&
+    LocalStorageUtility.hasItem(StorageKey.RetryAttempts) &&
+    LocalStorageUtility.hasItem(StorageKey.RetryInterval) &&
+    LocalStorageUtility.hasItem(StorageKey.MaxWaitTimeInSeconds)
   );
 };
 

src/Shared/StorageUtility.ts

@@ -6,6 +6,9 @@ export enum StorageKey {
   ActualItemPerPage,
   QueryTimeoutEnabled,
   QueryTimeout,
+  RetryAttempts,
+  RetryInterval,
+  MaxWaitTimeInSeconds,
   AutomaticallyCancelQueryAfterTimeout,
   ContainerPaginationEnabled,
   CustomItemPerPage,