From 547954c3dc2ea7c9562ee41482016b30d4f9f509 Mon Sep 17 00:00:00 2001 From: Asier Isayas Date: Thu, 30 Mar 2023 17:53:36 -0400 Subject: [PATCH] Lazy loading containers (#1411) Co-authored-by: Asier Isayas --- less/resourceTree.less | 8 ++- src/Common/Constants.ts | 2 +- src/Common/dataAccess/readCollections.ts | 30 ++++++++++ src/Contracts/DataModels.ts | 5 ++ src/Contracts/ViewModels.ts | 4 +- src/Explorer/Explorer.tsx | 2 +- .../Panes/SettingsPane/SettingsPane.tsx | 25 ++++++++ .../__snapshots__/SettingsPane.test.tsx.snap | 58 +++++++++++++++++++ src/Explorer/Tree/Database.tsx | 40 +++++++++++-- src/Explorer/Tree/ResourceTree.tsx | 12 ++++ src/Shared/StorageUtility.ts | 1 + 11 files changed, 175 insertions(+), 12 deletions(-) diff --git a/less/resourceTree.less b/less/resourceTree.less index 31410ab32..68aa3d0de 100644 --- a/less/resourceTree.less +++ b/less/resourceTree.less @@ -11,6 +11,10 @@ .collectionHeader { font-size: 12px; } + + .loadMoreHeader { + color: RGB(5, 99, 193); + } } .notebookResourceTree { @@ -23,6 +27,4 @@ .clickDisabled { pointer-events: none; } -} - - +} \ No newline at end of file diff --git a/src/Common/Constants.ts b/src/Common/Constants.ts index c9a458558..3feba8cc2 100644 --- a/src/Common/Constants.ts +++ b/src/Common/Constants.ts @@ -141,7 +141,7 @@ export class Queries { public static UnlimitedPageOption: string = "unlimited"; public static itemsPerPage: number = 100; public static unlimitedItemsPerPage: number = 100; // TODO: Figure out appropriate value so it works for accounts with a large number of partitions - + public static containersPerPage: number = 50; public static QueryEditorMinHeightRatio: number = 0.1; public static QueryEditorMaxHeightRatio: number = 0.4; public static readonly DefaultMaxDegreeOfParallelism = 6; diff --git a/src/Common/dataAccess/readCollections.ts b/src/Common/dataAccess/readCollections.ts index a5c8dd928..4a528b66c 100644 --- a/src/Common/dataAccess/readCollections.ts +++ b/src/Common/dataAccess/readCollections.ts @@ -1,3 +1,4 @@ +import { Queries } from "Common/Constants"; import { AuthType } from "../../AuthType"; import * as DataModels from "../../Contracts/DataModels"; import { userContext } from "../../UserContext"; @@ -31,6 +32,35 @@ export async function readCollections(databaseId: string): Promise { + const clearMessage = logConsoleProgress(`Querying containers for database ${databaseId}`); + try { + const sdkResponse = await client() + .database(databaseId) + .containers.query( + { query: "SELECT * FROM c" }, + { + continuationToken, + maxItemCount: Queries.containersPerPage, + } + ) + .fetchNext(); + const collectionsWithPagination: DataModels.CollectionsWithPagination = { + collections: sdkResponse.resources as DataModels.Collection[], + continuationToken: sdkResponse.continuationToken, + }; + return collectionsWithPagination; + } catch (error) { + handleError(error, "ReadCollections", `Error while querying containers for database ${databaseId}`); + throw error; + } finally { + clearMessage(); + } +} + async function readCollectionsWithARM(databaseId: string): Promise { let rpResponse; diff --git a/src/Contracts/DataModels.ts b/src/Contracts/DataModels.ts index c913310a3..e4f865c2c 100644 --- a/src/Contracts/DataModels.ts +++ b/src/Contracts/DataModels.ts @@ -156,6 +156,11 @@ export interface Collection extends Resource { requestSchema?: () => void; } +export interface CollectionsWithPagination { + continuationToken?: string; + collections?: Collection[]; +} + export interface Database extends Resource { collections?: Collection[]; } diff --git a/src/Contracts/ViewModels.ts b/src/Contracts/ViewModels.ts index 77ccc1cfe..c9bb351bc 100644 --- a/src/Contracts/ViewModels.ts +++ b/src/Contracts/ViewModels.ts @@ -87,13 +87,13 @@ export interface Database extends TreeNode { isDatabaseExpanded: ko.Observable; isDatabaseShared: ko.Computed; isSampleDB?: boolean; - + collectionsContinuationToken?: string; selectedSubnodeKind: ko.Observable; expandDatabase(): Promise; collapseDatabase(): void; - loadCollections(): Promise; + loadCollections(restart?: boolean): Promise; findCollectionWithId(collectionId: string): Collection; openAddCollection(database: Database, event: MouseEvent): void; onSettingsClick: () => void; diff --git a/src/Explorer/Explorer.tsx b/src/Explorer/Explorer.tsx index f1c689d47..f8ecb6923 100644 --- a/src/Explorer/Explorer.tsx +++ b/src/Explorer/Explorer.tsx @@ -577,7 +577,7 @@ export default class Explorer { try { await Promise.all( databasesToLoad.map(async (database: ViewModels.Database) => { - await database.loadCollections(); + await database.loadCollections(true); const isNewDatabase: boolean = _.some(newDatabases, (db: ViewModels.Database) => db.id() === database.id()); if (isNewDatabase) { database.expandDatabase(); diff --git a/src/Explorer/Panes/SettingsPane/SettingsPane.tsx b/src/Explorer/Panes/SettingsPane/SettingsPane.tsx index 9abc63f82..d7dced7a8 100644 --- a/src/Explorer/Panes/SettingsPane/SettingsPane.tsx +++ b/src/Explorer/Panes/SettingsPane/SettingsPane.tsx @@ -21,6 +21,11 @@ export const SettingsPane: FunctionComponent = () => { const [customItemPerPage, setCustomItemPerPage] = useState( LocalStorageUtility.getEntryNumber(StorageKey.CustomItemPerPage) || 0 ); + const [containerPaginationEnabled, setContainerPaginationEnabled] = useState( + LocalStorageUtility.hasItem(StorageKey.ContainerPaginationEnabled) + ? LocalStorageUtility.getEntryString(StorageKey.ContainerPaginationEnabled) === "true" + : false + ); const [crossPartitionQueryEnabled, setCrossPartitionQueryEnabled] = useState( LocalStorageUtility.hasItem(StorageKey.IsCrossPartitionQueryEnabled) ? LocalStorageUtility.getEntryString(StorageKey.IsCrossPartitionQueryEnabled) === "true" @@ -50,6 +55,7 @@ export const SettingsPane: FunctionComponent = () => { isCustomPageOptionSelected() ? customItemPerPage : Constants.Queries.unlimitedItemsPerPage ); LocalStorageUtility.setEntryNumber(StorageKey.CustomItemPerPage, customItemPerPage); + LocalStorageUtility.setEntryString(StorageKey.ContainerPaginationEnabled, containerPaginationEnabled.toString()); LocalStorageUtility.setEntryString(StorageKey.IsCrossPartitionQueryEnabled, crossPartitionQueryEnabled.toString()); LocalStorageUtility.setEntryNumber(StorageKey.MaxDegreeOfParellism, maxDegreeOfParallelism); @@ -185,6 +191,25 @@ export const SettingsPane: FunctionComponent = () => { )} +
+
+
+ Enable container pagination + + Load 50 containers at a time. Currently, containers are not pulled in alphanumeric order. + +
+ setContainerPaginationEnabled(!containerPaginationEnabled)} + /> +
+
{shouldShowCrossPartitionOption && (
diff --git a/src/Explorer/Panes/SettingsPane/__snapshots__/SettingsPane.test.tsx.snap b/src/Explorer/Panes/SettingsPane/__snapshots__/SettingsPane.test.tsx.snap index 928e7f7a7..ffa847681 100644 --- a/src/Explorer/Panes/SettingsPane/__snapshots__/SettingsPane.test.tsx.snap +++ b/src/Explorer/Panes/SettingsPane/__snapshots__/SettingsPane.test.tsx.snap @@ -97,6 +97,35 @@ exports[`Settings Pane should render Default properly 1`] = `
+
+
+
+ Enable container pagination + + Load 50 containers at a time. Currently, containers are not pulled in alphanumeric order. + +
+ +
+
@@ -182,6 +211,35 @@ exports[`Settings Pane should render Gremlin properly 1`] = `
+
+
+
+ Enable container pagination + + Load 50 containers at a time. Currently, containers are not pulled in alphanumeric order. + +
+ +
+
diff --git a/src/Explorer/Tree/Database.tsx b/src/Explorer/Tree/Database.tsx index a31b79b97..19cf82e31 100644 --- a/src/Explorer/Tree/Database.tsx +++ b/src/Explorer/Tree/Database.tsx @@ -3,7 +3,7 @@ import React from "react"; import * as _ from "underscore"; import { AuthType } from "../../AuthType"; import * as Constants from "../../Common/Constants"; -import { readCollections } from "../../Common/dataAccess/readCollections"; +import { readCollections, readCollectionsWithPagination } from "../../Common/dataAccess/readCollections"; import { readDatabaseOffer } from "../../Common/dataAccess/readDatabaseOffer"; import { getErrorMessage, getErrorStack } from "../../Common/ErrorHandlingUtils"; import * as Logger from "../../Common/Logger"; @@ -13,6 +13,7 @@ import * as ViewModels from "../../Contracts/ViewModels"; import { useSidePanel } from "../../hooks/useSidePanel"; import { useTabs } from "../../hooks/useTabs"; import { IJunoResponse, JunoClient } from "../../Juno/JunoClient"; +import * as StorageUtility from "../../Shared/StorageUtility"; import { Action, ActionModifiers } from "../../Shared/Telemetry/TelemetryConstants"; import * as TelemetryProcessor from "../../Shared/Telemetry/TelemetryProcessor"; import { userContext } from "../../UserContext"; @@ -38,6 +39,7 @@ export default class Database implements ViewModels.Database { public selectedSubnodeKind: ko.Observable; public junoClient: JunoClient; public isSampleDB: boolean; + public collectionsContinuationToken?: string; private isOfferRead: boolean; constructor(container: Explorer, data: DataModels.Database) { @@ -140,7 +142,11 @@ export default class Database implements ViewModels.Database { } await this.loadOffer(); - await this.loadCollections(); + + if (this.collections()?.length === 0) { + await this.loadCollections(true); + } + this.isDatabaseExpanded(true); TelemetryProcessor.trace(Action.ExpandTreeNode, ActionModifiers.Mark, { description: "Database node", @@ -162,9 +168,31 @@ export default class Database implements ViewModels.Database { }); } - public async loadCollections(): Promise { + public async loadCollections(restart = false) { const collectionVMs: Collection[] = []; - const collections: DataModels.Collection[] = await readCollections(this.id()); + let collections: DataModels.Collection[] = []; + if (restart) { + this.collectionsContinuationToken = undefined; + } + const containerPaginationEnabled = + StorageUtility.LocalStorageUtility.getEntryString(StorageUtility.StorageKey.ContainerPaginationEnabled) === + "true"; + if (containerPaginationEnabled) { + const collectionsWithPagination: DataModels.CollectionsWithPagination = await readCollectionsWithPagination( + this.id(), + this.collectionsContinuationToken + ); + + if (collectionsWithPagination.collections?.length === Constants.Queries.containersPerPage) { + this.collectionsContinuationToken = collectionsWithPagination.continuationToken; + } else { + this.collectionsContinuationToken = undefined; + } + collections = collectionsWithPagination.collections; + } else { + collections = await readCollections(this.id()); + } + // TODO Remove // This is a hack to make Mongo collections read via ARM have a SQL-ish partitionKey property if (userContext.apiType === "Mongo" && userContext.authType === AuthType.AAD) { @@ -199,7 +227,9 @@ export default class Database implements ViewModels.Database { //merge collections this.addCollectionsToList(collectionVMs); - this.deleteCollectionsFromList(deltaCollections.toDelete); + if (!containerPaginationEnabled || restart) { + this.deleteCollectionsFromList(deltaCollections.toDelete); + } useDatabases.getState().updateDatabase(this); } diff --git a/src/Explorer/Tree/ResourceTree.tsx b/src/Explorer/Tree/ResourceTree.tsx index 24eb34579..afb5a118e 100644 --- a/src/Explorer/Tree/ResourceTree.tsx +++ b/src/Explorer/Tree/ResourceTree.tsx @@ -479,6 +479,18 @@ export const ResourceTree: React.FC = ({ container }: Resourc databaseNode.children.push(buildCollectionNode(database, collection)) ); + if (database.collectionsContinuationToken) { + const loadMoreNode: TreeNode = { + label: "load more", + className: "loadMoreHeader", + onClick: async () => { + await database.loadCollections(); + useDatabases.getState().updateDatabase(database); + }, + }; + databaseNode.children.push(loadMoreNode); + } + database.collections.subscribe((collections: ViewModels.Collection[]) => { collections.forEach((collection: ViewModels.Collection) => databaseNode.children.push(buildCollectionNode(database, collection)) diff --git a/src/Shared/StorageUtility.ts b/src/Shared/StorageUtility.ts index 0d8477016..ad8a932e1 100644 --- a/src/Shared/StorageUtility.ts +++ b/src/Shared/StorageUtility.ts @@ -4,6 +4,7 @@ import * as SessionStorageUtility from "./SessionStorageUtility"; export { LocalStorageUtility, SessionStorageUtility }; export enum StorageKey { ActualItemPerPage, + ContainerPaginationEnabled, CustomItemPerPage, DatabaseAccountId, EncryptedKeyToken,