From 9cbf6325774163be4143b24de7ba4b94f9350767 Mon Sep 17 00:00:00 2001 From: victor-meng <56978073+victor-meng@users.noreply.github.com> Date: Fri, 20 Nov 2020 12:21:16 -0800 Subject: [PATCH] Get collection usage size with ARM metrics call (#327) - Removed `readCollectionQuotaInfo` call. The only data we need is the usage size which we can get via the ARM metrics call instead. - Added `getCollectionUsageSize` which fetches the `DataUsage` and `IndexUsage` metrics, converts them to KB, and returns the sum as the total usage size --- .../dataAccess/getCollectionDataUsageSize.ts | 87 +++++++++++++++++++ .../getIndexTransformationProgress.ts | 2 +- .../dataAccess/readCollectionQuotaInfo.ts | 45 ---------- src/Contracts/DataModels.ts | 12 --- src/Contracts/ViewModels.ts | 2 +- .../SettingsSubComponents/ScaleComponent.tsx | 2 +- .../ScaleComponent.test.tsx.snap | 1 + src/Explorer/Controls/Settings/TestUtils.tsx | 2 +- .../SettingsComponent.test.tsx.snap | 4 +- src/Explorer/Tree/Collection.test.ts | 5 +- src/Explorer/Tree/Collection.ts | 28 ++---- src/Explorer/Tree/Database.ts | 2 +- .../Tree/ResourceTreeAdapter.test.tsx | 4 +- src/Utils/arm/request.ts | 20 ++++- 14 files changed, 124 insertions(+), 92 deletions(-) create mode 100644 src/Common/dataAccess/getCollectionDataUsageSize.ts delete mode 100644 src/Common/dataAccess/readCollectionQuotaInfo.ts diff --git a/src/Common/dataAccess/getCollectionDataUsageSize.ts b/src/Common/dataAccess/getCollectionDataUsageSize.ts new file mode 100644 index 000000000..a59eab21a --- /dev/null +++ b/src/Common/dataAccess/getCollectionDataUsageSize.ts @@ -0,0 +1,87 @@ +import { armRequest } from "../../Utils/arm/request"; +import { configContext } from "../../ConfigContext"; +import { handleError } from "../ErrorHandlingUtils"; +import { userContext } from "../../UserContext"; + +interface TimeSeriesData { + data: { + timeStamp: string; + total: number; + }[]; + metadatavalues: { + name: { + localizedValue: string; + value: string; + }; + value: string; + }; +} + +interface MetricsData { + displayDescription: string; + errorCode: string; + id: string; + name: { + value: string; + localizedValue: string; + }; + timeseries: TimeSeriesData[]; + type: string; + unit: string; +} + +interface MetricsResponse { + cost: number; + interval: string; + namespace: string; + resourceregion: string; + timespan: string; + value: MetricsData[]; +} + +export const getCollectionUsageSizeInKB = async (databaseName: string, containerName: string): Promise => { + const subscriptionId = userContext.subscriptionId; + const resourceGroup = userContext.resourceGroup; + const accountName = userContext.databaseAccount.name; + const filter = `DatabaseName eq '${databaseName}' and CollectionName eq '${containerName}'`; + const metricNames = "DataUsage,IndexUsage"; + const path = `/subscriptions/${subscriptionId}/resourceGroups/${resourceGroup}/providers/Microsoft.DocumentDB/databaseAccounts/${accountName}/providers/microsoft.insights/metrics`; + + try { + const metricsResponse: MetricsResponse = await armRequest({ + host: configContext.ARM_ENDPOINT, + path, + method: "GET", + apiVersion: "2018-01-01", + queryParams: { + filter, + metricNames + } + }); + + if (metricsResponse?.value?.length !== 2) { + return undefined; + } + + const dataUsageData: MetricsData = metricsResponse.value[0]; + const indexUsagedata: MetricsData = metricsResponse.value[1]; + const dataUsageSizeInKb: number = getUsageSizeInKb(dataUsageData); + const indexUsageSizeInKb: number = getUsageSizeInKb(indexUsagedata); + + return dataUsageSizeInKb + indexUsageSizeInKb; + } catch (error) { + handleError(error, "getCollectionUsageSize"); + throw error; + } +}; + +const getUsageSizeInKb = (metricsData: MetricsData): number => { + if (metricsData?.errorCode !== "Success") { + throw Error(`Get collection usage size failed: ${metricsData.errorCode}`); + } + + const timeSeriesData: TimeSeriesData = metricsData?.timeseries?.[0]; + const usageSizeInBytes: number = timeSeriesData?.data?.[0]?.total; + + return usageSizeInBytes ? usageSizeInBytes / 1024 : 0; +}; diff --git a/src/Common/dataAccess/getIndexTransformationProgress.ts b/src/Common/dataAccess/getIndexTransformationProgress.ts index fa0298fbc..94dcf9fde 100644 --- a/src/Common/dataAccess/getIndexTransformationProgress.ts +++ b/src/Common/dataAccess/getIndexTransformationProgress.ts @@ -14,7 +14,7 @@ export async function getIndexTransformationProgress(databaseId: string, collect const response = await client() .database(databaseId) .container(collectionId) - .read({ populateQuotaInfo: true }); + .read(); indexTransformationPercentage = parseInt( response.headers[Constants.HttpHeaders.collectionIndexTransformationProgress] as string diff --git a/src/Common/dataAccess/readCollectionQuotaInfo.ts b/src/Common/dataAccess/readCollectionQuotaInfo.ts deleted file mode 100644 index 81565f821..000000000 --- a/src/Common/dataAccess/readCollectionQuotaInfo.ts +++ /dev/null @@ -1,45 +0,0 @@ -import * as DataModels from "../../Contracts/DataModels"; -import * as HeadersUtility from "../HeadersUtility"; -import * as ViewModels from "../../Contracts/ViewModels"; -import { ContainerDefinition, Resource } from "@azure/cosmos"; -import { HttpHeaders } from "../Constants"; -import { RequestOptions } from "@azure/cosmos/dist-esm"; -import { client } from "../CosmosClient"; -import { handleError } from "../ErrorHandlingUtils"; -import { logConsoleProgress } from "../../Utils/NotificationConsoleUtils"; - -interface ResourceWithStatistics { - statistics: DataModels.Statistic[]; -} - -export const readCollectionQuotaInfo = async ( - collection: ViewModels.Collection -): Promise => { - const clearMessage = logConsoleProgress(`Querying containers for database ${collection.id}`); - const options: RequestOptions = {}; - options.populateQuotaInfo = true; - options.initialHeaders = options.initialHeaders || {}; - options.initialHeaders[HttpHeaders.populatePartitionStatistics] = true; - - try { - const response = await client() - .database(collection.databaseId) - .container(collection.id()) - .read(options); - const quota: DataModels.CollectionQuotaInfo = HeadersUtility.getQuota(response.headers); - const resource = response.resource as ContainerDefinition & Resource & ResourceWithStatistics; - quota["usageSizeInKB"] = resource.statistics.reduce( - (previousValue: number, currentValue: DataModels.Statistic) => previousValue + currentValue.sizeInKB, - 0 - ); - quota["numPartitions"] = resource.statistics.length; - quota["uniqueKeyPolicy"] = collection.uniqueKeyPolicy; // TODO: Remove after refactoring (#119617) - - return quota; - } catch (error) { - handleError(error, "ReadCollectionQuotaInfo", `Error while querying quota info for container ${collection.id}`); - throw error; - } finally { - clearMessage(); - } -}; diff --git a/src/Contracts/DataModels.ts b/src/Contracts/DataModels.ts index 350330176..5ec15c572 100644 --- a/src/Contracts/DataModels.ts +++ b/src/Contracts/DataModels.ts @@ -230,18 +230,6 @@ export interface SDKOfferDefinition extends Resource { offerResourceId?: string; } -export interface CollectionQuotaInfo { - storedProcedures: number; - triggers: number; - functions: number; - documentsSize: number; - collectionSize: number; - documentsCount: number; - usageSizeInKB: number; - numPartitions: number; - uniqueKeyPolicy?: UniqueKeyPolicy; // TODO: This should ideally not be a part of the collection quota. Remove after refactoring. (#119617) -} - export interface OfferThroughputInfo { minimumRUForCollection: number; numPhysicalPartitions: number; diff --git a/src/Contracts/ViewModels.ts b/src/Contracts/ViewModels.ts index f0a0e45e5..a023f2fc4 100644 --- a/src/Contracts/ViewModels.ts +++ b/src/Contracts/ViewModels.ts @@ -120,7 +120,7 @@ export interface Collection extends CollectionBase { requestSchema?: () => void; indexingPolicy: ko.Observable; uniqueKeyPolicy: DataModels.UniqueKeyPolicy; - quotaInfo: ko.Observable; + usageSizeInKB: ko.Observable; offer: ko.Observable; conflictResolutionPolicy: ko.Observable; changeFeedPolicy: ko.Observable; diff --git a/src/Explorer/Controls/Settings/SettingsSubComponents/ScaleComponent.tsx b/src/Explorer/Controls/Settings/SettingsSubComponents/ScaleComponent.tsx index 47e0a4b65..c8f193b44 100644 --- a/src/Explorer/Controls/Settings/SettingsSubComponents/ScaleComponent.tsx +++ b/src/Explorer/Controls/Settings/SettingsSubComponents/ScaleComponent.tsx @@ -186,7 +186,7 @@ export class ScaleComponent extends React.Component { onScaleSaveableChange={this.props.onScaleSaveableChange} onScaleDiscardableChange={this.props.onScaleDiscardableChange} getThroughputWarningMessage={this.getThroughputWarningMessage} - usageSizeInKB={this.props.collection.quotaInfo().usageSizeInKB} + usageSizeInKB={this.props.collection.usageSizeInKB()} /> ); diff --git a/src/Explorer/Controls/Settings/SettingsSubComponents/__snapshots__/ScaleComponent.test.tsx.snap b/src/Explorer/Controls/Settings/SettingsSubComponents/__snapshots__/ScaleComponent.test.tsx.snap index e47b3cb2e..d498de8e7 100644 --- a/src/Explorer/Controls/Settings/SettingsSubComponents/__snapshots__/ScaleComponent.test.tsx.snap +++ b/src/Explorer/Controls/Settings/SettingsSubComponents/__snapshots__/ScaleComponent.test.tsx.snap @@ -58,6 +58,7 @@ exports[`ScaleComponent renders with correct initial notification 1`] = ` spendAckChecked={false} throughput={1000} throughputBaseline={1000} + usageSizeInKB={100} wasAutopilotOriginallySet={true} /> ({} as DataModels.CollectionQuotaInfo), + usageSizeInKB: ko.observable(100), offer: ko.observable({ autoscaleMaxThroughput: undefined, manualThroughput: 10000, diff --git a/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap b/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap index ee5174ba4..c4923e5d0 100644 --- a/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap +++ b/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap @@ -1300,9 +1300,9 @@ exports[`SettingsComponent renders 1`] = ` "version": 2, }, "partitionKeyProperty": "partitionKey", - "quotaInfo": [Function], "readSettings": [Function], "uniqueKeyPolicy": Object {}, + "usageSizeInKB": [Function], } } container={ @@ -3871,9 +3871,9 @@ exports[`SettingsComponent renders 1`] = ` "version": 2, }, "partitionKeyProperty": "partitionKey", - "quotaInfo": [Function], "readSettings": [Function], "uniqueKeyPolicy": Object {}, + "usageSizeInKB": [Function], } } container={ diff --git a/src/Explorer/Tree/Collection.test.ts b/src/Explorer/Tree/Collection.test.ts index c95d41e94..58d3be7bb 100644 --- a/src/Explorer/Tree/Collection.test.ts +++ b/src/Explorer/Tree/Collection.test.ts @@ -10,10 +10,9 @@ describe("Collection", () => { container: Explorer, databaseId: string, data: DataModels.Collection, - quotaInfo: DataModels.CollectionQuotaInfo, offer: DataModels.Offer ): Collection { - return new Collection(container, databaseId, data, quotaInfo, offer); + return new Collection(container, databaseId, data); } function generateMockCollectionsDataModelWithPartitionKey( @@ -50,7 +49,7 @@ describe("Collection", () => { }); mockContainer.deleteCollectionText = ko.observable("delete collection"); - return generateCollection(mockContainer, "abc", data, {} as DataModels.CollectionQuotaInfo, {} as DataModels.Offer); + return generateCollection(mockContainer, "abc", data, {} as DataModels.Offer); } describe("Partition key path parsing", () => { diff --git a/src/Explorer/Tree/Collection.ts b/src/Explorer/Tree/Collection.ts index c2579ef7b..14d0f68bb 100644 --- a/src/Explorer/Tree/Collection.ts +++ b/src/Explorer/Tree/Collection.ts @@ -10,7 +10,7 @@ import { readTriggers } from "../../Common/dataAccess/readTriggers"; import { readUserDefinedFunctions } from "../../Common/dataAccess/readUserDefinedFunctions"; import { createDocument } from "../../Common/DocumentClientUtilityBase"; import { readCollectionOffer } from "../../Common/dataAccess/readCollectionOffer"; -import { readCollectionQuotaInfo } from "../../Common/dataAccess/readCollectionQuotaInfo"; +import { getCollectionUsageSizeInKB } from "../../Common/dataAccess/getCollectionDataUsageSize"; import * as Logger from "../../Common/Logger"; import * as DataModels from "../../Contracts/DataModels"; import * as ViewModels from "../../Contracts/ViewModels"; @@ -37,7 +37,6 @@ import UserDefinedFunction from "./UserDefinedFunction"; import { configContext, Platform } from "../../ConfigContext"; import Explorer from "../Explorer"; import { userContext } from "../../UserContext"; -import TabsBase from "../Tabs/TabsBase"; import { fetchPortalNotifications } from "../../Common/PortalNotifications"; import { getErrorMessage, getErrorStack } from "../../Common/ErrorHandlingUtils"; @@ -54,7 +53,8 @@ export default class Collection implements ViewModels.Collection { public defaultTtl: ko.Observable; public indexingPolicy: ko.Observable; public uniqueKeyPolicy: DataModels.UniqueKeyPolicy; - public quotaInfo: ko.Observable; + public usageSizeInKB: ko.Observable; + public offer: ko.Observable; public conflictResolutionPolicy: ko.Observable; public changeFeedPolicy: ko.Observable; @@ -95,13 +95,7 @@ export default class Collection implements ViewModels.Collection { public userDefinedFunctionsFocused: ko.Observable; public triggersFocused: ko.Observable; - constructor( - container: Explorer, - databaseId: string, - data: DataModels.Collection, - quotaInfo: DataModels.CollectionQuotaInfo, - offer: DataModels.Offer - ) { + constructor(container: Explorer, databaseId: string, data: DataModels.Collection) { this.nodeKind = "Collection"; this.container = container; this.self = data._self; @@ -113,8 +107,8 @@ export default class Collection implements ViewModels.Collection { this.id = ko.observable(data.id); this.defaultTtl = ko.observable(data.defaultTtl); this.indexingPolicy = ko.observable(data.indexingPolicy); - this.quotaInfo = ko.observable(quotaInfo); - this.offer = ko.observable(offer); + this.usageSizeInKB = ko.observable(); + this.offer = ko.observable(); this.conflictResolutionPolicy = ko.observable(data.conflictResolutionPolicy); this.changeFeedPolicy = ko.observable(data.changeFeedPolicy); this.analyticalStorageTtl = ko.observable(data.analyticalStorageTtl); @@ -607,14 +601,6 @@ export default class Collection implements ViewModels.Collection { } }; - private async loadCollectionQuotaInfo(): Promise { - // TODO: Use the collection entity cache to get quota info - const quotaInfoWithUniqueKeyPolicy = await readCollectionQuotaInfo(this); - this.uniqueKeyPolicy = quotaInfoWithUniqueKeyPolicy.uniqueKeyPolicy; - const quotaInfo = _.omit(quotaInfoWithUniqueKeyPolicy, "uniqueKeyPolicy"); - this.quotaInfo(quotaInfo); - } - public onNewQueryClick(source: any, event: MouseEvent, queryText?: string) { const collection: ViewModels.Collection = source.collection || source; const id = this.container.tabsManager.getTabs(ViewModels.CollectionTabKind.Query).length + 1; @@ -1287,7 +1273,7 @@ export default class Collection implements ViewModels.Collection { try { this.offer(await readCollectionOffer(params)); - await this.loadCollectionQuotaInfo(); + this.usageSizeInKB(await getCollectionUsageSizeInKB(this.databaseId, this.id())); TelemetryProcessor.traceSuccess( Action.LoadOffers, diff --git a/src/Explorer/Tree/Database.ts b/src/Explorer/Tree/Database.ts index cf6c48c08..c160d44b8 100644 --- a/src/Explorer/Tree/Database.ts +++ b/src/Explorer/Tree/Database.ts @@ -193,7 +193,7 @@ export default class Database implements ViewModels.Database { }); deltaCollections.toAdd.forEach((collection: DataModels.Collection) => { - const collectionVM: Collection = new Collection(this.container, this.id(), collection, null, null); + const collectionVM: Collection = new Collection(this.container, this.id(), collection); collectionVMs.push(collectionVM); }); diff --git a/src/Explorer/Tree/ResourceTreeAdapter.test.tsx b/src/Explorer/Tree/ResourceTreeAdapter.test.tsx index 790f1b3e1..076c7a0b0 100644 --- a/src/Explorer/Tree/ResourceTreeAdapter.test.tsx +++ b/src/Explorer/Tree/ResourceTreeAdapter.test.tsx @@ -229,9 +229,7 @@ const createMockCollection = (): ViewModels.Collection => { const mockCollectionVM: ViewModels.Collection = new Collection( createMockContainer(), "fakeDatabaseId", - mockCollection, - undefined, - undefined + mockCollection ); return mockCollectionVM; diff --git a/src/Utils/arm/request.ts b/src/Utils/arm/request.ts index 443ec9c44..23229d46b 100644 --- a/src/Utils/arm/request.ts +++ b/src/Utils/arm/request.ts @@ -33,18 +33,36 @@ export class ARMError extends Error { public code: string | number; } +interface ARMQueryParams { + filter?: string; + metricNames?: string; +} + interface Options { host: string; path: string; apiVersion: string; method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD"; body?: unknown; + queryParams?: ARMQueryParams; } // TODO: This is very similar to what is happening in ResourceProviderClient.ts. Should probably merge them. -export async function armRequest({ host, path, apiVersion, method, body: requestBody }: Options): Promise { +export async function armRequest({ + host, + path, + apiVersion, + method, + body: requestBody, + queryParams +}: Options): Promise { const url = new URL(path, host); url.searchParams.append("api-version", configContext.armAPIVersion || apiVersion); + if (queryParams) { + queryParams.filter && url.searchParams.append("$filter", queryParams.filter); + queryParams.metricNames && url.searchParams.append("metricnames", queryParams.metricNames); + } + const response = await window.fetch(url.href, { method, headers: {