Lazy loading containers (#1411)

Co-authored-by: Asier Isayas <aisayas@microsoft.com>

src/Explorer/Explorer.tsx

@@ -577,7 +577,7 @@ export default class Explorer {
     try {
       await Promise.all(
         databasesToLoad.map(async (database: ViewModels.Database) => {
-          await database.loadCollections();
+          await database.loadCollections(true);
           const isNewDatabase: boolean = _.some(newDatabases, (db: ViewModels.Database) => db.id() === database.id());
           if (isNewDatabase) {
             database.expandDatabase();

src/Explorer/Panes/SettingsPane/SettingsPane.tsx

@@ -21,6 +21,11 @@ export const SettingsPane: FunctionComponent = () => {
   const [customItemPerPage, setCustomItemPerPage] = useState<number>(
     LocalStorageUtility.getEntryNumber(StorageKey.CustomItemPerPage) || 0
   );
+  const [containerPaginationEnabled, setContainerPaginationEnabled] = useState<boolean>(
+    LocalStorageUtility.hasItem(StorageKey.ContainerPaginationEnabled)
+      ? LocalStorageUtility.getEntryString(StorageKey.ContainerPaginationEnabled) === "true"
+      : false
+  );
   const [crossPartitionQueryEnabled, setCrossPartitionQueryEnabled] = useState<boolean>(
     LocalStorageUtility.hasItem(StorageKey.IsCrossPartitionQueryEnabled)
       ? LocalStorageUtility.getEntryString(StorageKey.IsCrossPartitionQueryEnabled) === "true"
@@ -50,6 +55,7 @@ export const SettingsPane: FunctionComponent = () => {
       isCustomPageOptionSelected() ? customItemPerPage : Constants.Queries.unlimitedItemsPerPage
     );
     LocalStorageUtility.setEntryNumber(StorageKey.CustomItemPerPage, customItemPerPage);
+    LocalStorageUtility.setEntryString(StorageKey.ContainerPaginationEnabled, containerPaginationEnabled.toString());
     LocalStorageUtility.setEntryString(StorageKey.IsCrossPartitionQueryEnabled, crossPartitionQueryEnabled.toString());
     LocalStorageUtility.setEntryNumber(StorageKey.MaxDegreeOfParellism, maxDegreeOfParallelism);
 
@@ -185,6 +191,25 @@ export const SettingsPane: FunctionComponent = () => {
             </div>
           </div>
         )}
+        <div className="settingsSection">
+          <div className="settingsSectionPart">
+            <div className="settingsSectionLabel">
+              Enable container pagination
+              <InfoTooltip>
+                Load 50 containers at a time. Currently, containers are not pulled in alphanumeric order.
+              </InfoTooltip>
+            </div>
+            <Checkbox
+              styles={{
+                label: { padding: 0 },
+              }}
+              className="padding"
+              ariaLabel="Enable container pagination"
+              checked={containerPaginationEnabled}
+              onChange={() => setContainerPaginationEnabled(!containerPaginationEnabled)}
+            />
+          </div>
+        </div>
         {shouldShowCrossPartitionOption && (
           <div className="settingsSection">
             <div className="settingsSectionPart">

src/Explorer/Panes/SettingsPane/__snapshots__/SettingsPane.test.tsx.snap

@@ -97,6 +97,35 @@ exports[`Settings Pane should render Default properly 1`] = `
         </div>
       </div>
     </div>
+    <div
+      className="settingsSection"
+    >
+      <div
+        className="settingsSectionPart"
+      >
+        <div
+          className="settingsSectionLabel"
+        >
+          Enable container pagination
+          <InfoTooltip>
+            Load 50 containers at a time. Currently, containers are not pulled in alphanumeric order.
+          </InfoTooltip>
+        </div>
+        <StyledCheckboxBase
+          ariaLabel="Enable container pagination"
+          checked={false}
+          className="padding"
+          onChange={[Function]}
+          styles={
+            Object {
+              "label": Object {
+                "padding": 0,
+              },
+            }
+          }
+        />
+      </div>
+    </div>
     <div
       className="settingsSection"
     >
@@ -182,6 +211,35 @@ exports[`Settings Pane should render Gremlin properly 1`] = `
   <div
     className="paneMainContent"
   >
+    <div
+      className="settingsSection"
+    >
+      <div
+        className="settingsSectionPart"
+      >
+        <div
+          className="settingsSectionLabel"
+        >
+          Enable container pagination
+          <InfoTooltip>
+            Load 50 containers at a time. Currently, containers are not pulled in alphanumeric order.
+          </InfoTooltip>
+        </div>
+        <StyledCheckboxBase
+          ariaLabel="Enable container pagination"
+          checked={false}
+          className="padding"
+          onChange={[Function]}
+          styles={
+            Object {
+              "label": Object {
+                "padding": 0,
+              },
+            }
+          }
+        />
+      </div>
+    </div>
     <div
       className="settingsSection"
     >

src/Explorer/Tree/Database.tsx

@@ -3,7 +3,7 @@ import React from "react";
 import * as _ from "underscore";
 import { AuthType } from "../../AuthType";
 import * as Constants from "../../Common/Constants";
-import { readCollections } from "../../Common/dataAccess/readCollections";
+import { readCollections, readCollectionsWithPagination } from "../../Common/dataAccess/readCollections";
 import { readDatabaseOffer } from "../../Common/dataAccess/readDatabaseOffer";
 import { getErrorMessage, getErrorStack } from "../../Common/ErrorHandlingUtils";
 import * as Logger from "../../Common/Logger";
@@ -13,6 +13,7 @@ import * as ViewModels from "../../Contracts/ViewModels";
 import { useSidePanel } from "../../hooks/useSidePanel";
 import { useTabs } from "../../hooks/useTabs";
 import { IJunoResponse, JunoClient } from "../../Juno/JunoClient";
+import * as StorageUtility from "../../Shared/StorageUtility";
 import { Action, ActionModifiers } from "../../Shared/Telemetry/TelemetryConstants";
 import * as TelemetryProcessor from "../../Shared/Telemetry/TelemetryProcessor";
 import { userContext } from "../../UserContext";
@@ -38,6 +39,7 @@ export default class Database implements ViewModels.Database {
   public selectedSubnodeKind: ko.Observable<ViewModels.CollectionTabKind>;
   public junoClient: JunoClient;
   public isSampleDB: boolean;
+  public collectionsContinuationToken?: string;
   private isOfferRead: boolean;
 
   constructor(container: Explorer, data: DataModels.Database) {
@@ -140,7 +142,11 @@ export default class Database implements ViewModels.Database {
     }
 
     await this.loadOffer();
-    await this.loadCollections();
+
+    if (this.collections()?.length === 0) {
+      await this.loadCollections(true);
+    }
+
     this.isDatabaseExpanded(true);
     TelemetryProcessor.trace(Action.ExpandTreeNode, ActionModifiers.Mark, {
       description: "Database node",
@@ -162,9 +168,31 @@ export default class Database implements ViewModels.Database {
     });
   }
 
-  public async loadCollections(): Promise<void> {
+  public async loadCollections(restart = false) {
     const collectionVMs: Collection[] = [];
-    const collections: DataModels.Collection[] = await readCollections(this.id());
+    let collections: DataModels.Collection[] = [];
+    if (restart) {
+      this.collectionsContinuationToken = undefined;
+    }
+    const containerPaginationEnabled =
+      StorageUtility.LocalStorageUtility.getEntryString(StorageUtility.StorageKey.ContainerPaginationEnabled) ===
+      "true";
+    if (containerPaginationEnabled) {
+      const collectionsWithPagination: DataModels.CollectionsWithPagination = await readCollectionsWithPagination(
+        this.id(),
+        this.collectionsContinuationToken
+      );
+
+      if (collectionsWithPagination.collections?.length === Constants.Queries.containersPerPage) {
+        this.collectionsContinuationToken = collectionsWithPagination.continuationToken;
+      } else {
+        this.collectionsContinuationToken = undefined;
+      }
+      collections = collectionsWithPagination.collections;
+    } else {
+      collections = await readCollections(this.id());
+    }
+
     // TODO Remove
     // This is a hack to make Mongo collections read via ARM have a SQL-ish partitionKey property
     if (userContext.apiType === "Mongo" && userContext.authType === AuthType.AAD) {
@@ -199,7 +227,9 @@ export default class Database implements ViewModels.Database {
 
     //merge collections
     this.addCollectionsToList(collectionVMs);
-    this.deleteCollectionsFromList(deltaCollections.toDelete);
+    if (!containerPaginationEnabled || restart) {
+      this.deleteCollectionsFromList(deltaCollections.toDelete);
+    }
 
     useDatabases.getState().updateDatabase(this);
   }

src/Explorer/Tree/ResourceTree.tsx

@@ -479,6 +479,18 @@ export const ResourceTree: React.FC<ResourceTreeProps> = ({ container }: Resourc
           databaseNode.children.push(buildCollectionNode(database, collection))
         );
 
+      if (database.collectionsContinuationToken) {
+        const loadMoreNode: TreeNode = {
+          label: "load more",
+          className: "loadMoreHeader",
+          onClick: async () => {
+            await database.loadCollections();
+            useDatabases.getState().updateDatabase(database);
+          },
+        };
+        databaseNode.children.push(loadMoreNode);
+      }
+
       database.collections.subscribe((collections: ViewModels.Collection[]) => {
         collections.forEach((collection: ViewModels.Collection) =>
           databaseNode.children.push(buildCollectionNode(database, collection))