From 4e8410cd66d4abc7984db12707df001068450a56 Mon Sep 17 00:00:00 2001 From: sunghyunkang1111 <114709653+sunghyunkang1111@users.noreply.github.com> Date: Fri, 1 May 2026 12:38:53 -0500 Subject: [PATCH] Added quantizerType support in vector embedding (#2471) * Added quantizerType support in vector embedding * Fix build issues * Address PR Comments * address pr comments --- src/Contracts/DataModels.ts | 1 + .../Controls/Settings/SettingsComponent.tsx | 18 +++- .../ContainerPolicyComponent.tsx | 22 ++++- .../Controls/Settings/SettingsUtils.tsx | 3 +- .../SettingsComponent.test.tsx.snap | 3 + .../VectorEmbeddingPoliciesComponent.tsx | 92 ++++++++++++++----- .../VectorSearch/VectorSearchUtils.ts | 9 ++ src/Localization/en/Resources.json | 26 +++++- 8 files changed, 143 insertions(+), 31 deletions(-) diff --git a/src/Contracts/DataModels.ts b/src/Contracts/DataModels.ts index 0a51d721a..39006ab76 100644 --- a/src/Contracts/DataModels.ts +++ b/src/Contracts/DataModels.ts @@ -255,6 +255,7 @@ export interface VectorIndex { vectorIndexShardKey?: string[]; indexingSearchListSize?: number; quantizationByteSize?: number; + quantizerType?: "product" | "spherical"; } export interface FullTextIndex { diff --git a/src/Explorer/Controls/Settings/SettingsComponent.tsx b/src/Explorer/Controls/Settings/SettingsComponent.tsx index bc269cb4b..4aca3ce6b 100644 --- a/src/Explorer/Controls/Settings/SettingsComponent.tsx +++ b/src/Explorer/Controls/Settings/SettingsComponent.tsx @@ -15,6 +15,7 @@ import { } from "Explorer/Controls/Settings/SettingsSubComponents/ThroughputInputComponents/ThroughputBucketsComponent"; import { useIndexingPolicyStore } from "Explorer/Tabs/QueryTab/ResultsView"; import { useDatabases } from "Explorer/useDatabases"; +import { Keys, t } from "Localization"; import { isFabricNative } from "Platform/Fabric/FabricUtil"; import { isVectorSearchEnabled } from "Utils/CapabilityUtils"; import { isRunningOnPublicCloud } from "Utils/CloudUtils"; @@ -44,7 +45,6 @@ import { useCommandBar } from "../../Menus/CommandBar/CommandBarComponentAdapter import { SettingsTabV2 } from "../../Tabs/SettingsTabV2"; import "./SettingsComponent.less"; import { mongoIndexingPolicyAADError } from "./SettingsRenderUtils"; -import { Keys, t } from "Localization"; import { ConflictResolutionComponent, ConflictResolutionComponentProps, @@ -555,6 +555,19 @@ export class SettingsComponent extends React.Component this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy }); + private onVectorIndexesChange = (newVectorIndexes: DataModels.VectorIndex[]): void => { + const currentIndexingPolicy: DataModels.IndexingPolicy = + this.state.indexingPolicyContent || ({} as DataModels.IndexingPolicy); + const newIndexingPolicy: DataModels.IndexingPolicy = { + ...currentIndexingPolicy, + vectorIndexes: newVectorIndexes, + }; + this.setState({ + indexingPolicyContent: newIndexingPolicy, + isIndexingPolicyDirty: true, + }); + }; + private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void => this.setState({ fullTextPolicy: newFullTextPolicy }); @@ -1332,6 +1345,9 @@ export class SettingsComponent extends React.Component void; onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void; onVectorEmbeddingPolicyValidationChange: (isValid: boolean) => void; + vectorIndexes: VectorIndex[]; + vectorIndexesBaseline: VectorIndex[]; + onVectorIndexesChange: (newVectorIndexes: VectorIndex[]) => void; isVectorSearchEnabled: boolean; fullTextPolicy: FullTextPolicy; fullTextPolicyBaseline: FullTextPolicy; @@ -33,6 +36,9 @@ export const ContainerPolicyComponent: React.FC = onVectorEmbeddingPolicyChange, onVectorEmbeddingPolicyDirtyChange, onVectorEmbeddingPolicyValidationChange, + vectorIndexes, + vectorIndexesBaseline, + onVectorIndexesChange, isVectorSearchEnabled, fullTextPolicy, fullTextPolicyBaseline, @@ -78,6 +84,7 @@ export const ContainerPolicyComponent: React.FC = const checkAndSendVectorEmbeddingPoliciesToSettings = ( newVectorEmbeddings: VectorEmbedding[], + newVectorIndexes: VectorIndex[], validationPassed: boolean, ): void => { onVectorEmbeddingPolicyValidationChange(validationPassed); @@ -86,6 +93,9 @@ export const ContainerPolicyComponent: React.FC = if (isVectorDirty) { onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings }); } + if (isDirty(newVectorIndexes ?? [], vectorIndexesBaseline ?? [])) { + onVectorIndexesChange(newVectorIndexes); + } }; const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => { @@ -169,12 +179,14 @@ export const ContainerPolicyComponent: React.FC = checkAndSendVectorEmbeddingPoliciesToSettings(vectorEmbeddings, validationPassed)} + ) => + checkAndSendVectorEmbeddingPoliciesToSettings(newVectorEmbeddings, newVectorIndexes, validationPassed) + } discardChanges={discardVectorChanges} onChangesDiscarded={onVectorChangesDiscarded} /> diff --git a/src/Explorer/Controls/Settings/SettingsUtils.tsx b/src/Explorer/Controls/Settings/SettingsUtils.tsx index f445caefd..6e0990849 100644 --- a/src/Explorer/Controls/Settings/SettingsUtils.tsx +++ b/src/Explorer/Controls/Settings/SettingsUtils.tsx @@ -1,7 +1,7 @@ +import { Keys, t } from "Localization"; import * as Constants from "../../../Common/Constants"; import * as DataModels from "../../../Contracts/DataModels"; import * as ViewModels from "../../../Contracts/ViewModels"; -import { Keys, t } from "Localization"; import { isFabricNative } from "../../../Platform/Fabric/FabricUtil"; import { userContext } from "../../../UserContext"; import { isCapabilityEnabled } from "../../../Utils/CapabilityUtils"; @@ -15,6 +15,7 @@ export type isDirtyTypes = | DataModels.IndexingPolicy | DataModels.ComputedProperties | DataModels.VectorEmbedding[] + | DataModels.VectorIndex[] | DataModels.FullTextPolicy | DataModels.ThroughputBucket[] | DataModels.DataMaskingPolicy; diff --git a/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap b/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap index daa26da90..819858641 100644 --- a/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap +++ b/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap @@ -359,10 +359,13 @@ exports[`SettingsComponent renders 1`] = ` onVectorEmbeddingPolicyChange={[Function]} onVectorEmbeddingPolicyDirtyChange={[Function]} onVectorEmbeddingPolicyValidationChange={[Function]} + onVectorIndexesChange={[Function]} resetShouldDiscardContainerPolicyChange={[Function]} shouldDiscardContainerPolicies={false} vectorEmbeddingPolicy={{}} vectorEmbeddingPolicyBaseline={{}} + vectorIndexes={[]} + vectorIndexesBaseline={[]} /> diff --git a/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx b/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx index ba2610fb4..88a6f021b 100644 --- a/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx +++ b/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx @@ -16,7 +16,10 @@ import { getDataTypeOptions, getDistanceFunctionOptions, getIndexTypeOptions, + getQuantizerTypeOptions, + supportsQuantization, } from "Explorer/Controls/VectorSearch/VectorSearchUtils"; +import { Keys, t } from "Localization"; import React, { FunctionComponent, useState } from "react"; export interface IVectorEmbeddingPoliciesComponentProps { @@ -46,6 +49,7 @@ export interface VectorEmbeddingPolicyData { indexingSearchListSizeError?: string; quantizationByteSize?: number; quantizationByteSizeError?: string; + quantizerType?: VectorIndex["quantizerType"]; } type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType"; @@ -110,7 +114,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { let error = ""; if (!path) { - error = "Path should not be empty"; + error = t(Keys.controls.vectorEmbeddingPolicies.pathEmptyError); } if ( index >= 0 && @@ -119,7 +123,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { let error = ""; if (dimension <= 0 || dimension > 4096) { - error = "Dimension must be greater than 0 and less than or equal 4096"; + error = t(Keys.controls.vectorEmbeddingPolicies.dimensionRangeError); } if (indexType === "flat" && dimension > 505) { - error = "Maximum allowed dimension for flat index is 505"; + error = t(Keys.controls.vectorEmbeddingPolicies.dimensionFlatIndexError); } return error; }; @@ -138,7 +142,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { let error = ""; if (size < 1 || size > 512) { - error = "Quantization byte size must be greater than 0 and less than or equal to 512"; + error = t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeRangeError); } return error; }; @@ -146,7 +150,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { let error = ""; if (size < 25 || size > 500) { - error = "Indexing search list size must be greater than or equal to 25 and less than or equal to 500"; + error = t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSizeRangeError); } return error; }; @@ -155,11 +159,14 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined; + const matchingType = matchingIndex?.type; + const supportsQuantizer = supportsQuantization(matchingType); mergedData.push({ ...embedding, - indexType: matchingIndex?.type || "none", + indexType: matchingType || "none", indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined, quantizationByteSize: matchingIndex?.quantizationByteSize || undefined, + quantizerType: supportsQuantizer ? matchingIndex?.quantizerType || "product" : undefined, vectorIndexShardKey: matchingIndex?.vectorIndexShardKey || undefined, pathError: onVectorEmbeddingPathError(embedding.path), dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"), @@ -202,6 +209,9 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + vectorEmbeddings[index].quantizerType = option.key as VectorIndex["quantizerType"]; setVectorEmbeddingPolicyData(vectorEmbeddings); }; @@ -306,8 +327,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent { - const containerName: string = isGlobalSecondaryIndex ? "global secondary index" : "container"; - return `This is dynamically set by the ${containerName} if left blank, or it can be set to a fixed number`; + const containerName = isGlobalSecondaryIndex + ? t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipGlobalSecondaryIndexName) + : t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipContainerName); + return t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltip, { containerName }); }; return ( @@ -319,7 +342,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent onDelete(index)} disableDelete={false} @@ -337,7 +360,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent - Quantization byte size + {t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSize)} {getQuantizationByteSizeTooltipContent()} + + + , option: IDropdownOption) => + onQuantizerTypeChange(index, option) + } + /> + - Vector index shard key + {t(Keys.controls.vectorEmbeddingPolicies.vectorIndexShardKey)} ))} - Add vector embedding + {t(Keys.controls.vectorEmbeddingPolicies.addVectorEmbedding)} ); diff --git a/src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts b/src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts index 745a6cce0..7e90053fc 100644 --- a/src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts +++ b/src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts @@ -1,4 +1,6 @@ import { IDropdownOption } from "@fluentui/react"; +import { VectorIndex } from "Contracts/DataModels"; +import { Keys, t } from "Localization"; const dataTypes = ["float32", "uint8", "int8", "float16"]; const distanceFunctions = ["euclidean", "cosine", "dotproduct"]; @@ -7,6 +9,13 @@ const indexTypes = ["none", "flat", "diskANN", "quantizedFlat"]; export const getDataTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(dataTypes); export const getDistanceFunctionOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(distanceFunctions); export const getIndexTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(indexTypes); +export const getQuantizerTypeOptions = (): IDropdownOption[] => [ + { key: "product", text: "Product" }, + { key: "spherical", text: `Spherical (${t(Keys.common.preview)})` }, +]; + +export const supportsQuantization = (indexType: VectorIndex["type"] | "none" | undefined): boolean => + indexType === "quantizedFlat" || indexType === "diskANN"; function createDropdownOptionsFromLiterals(literals: T[]): IDropdownOption[] { return literals.map((value) => ({ diff --git a/src/Localization/en/Resources.json b/src/Localization/en/Resources.json index 8278207fe..2d80a2286 100644 --- a/src/Localization/en/Resources.json +++ b/src/Localization/en/Resources.json @@ -33,7 +33,8 @@ "publish": "Publish", "browse": "Browse", "increaseValueBy1": "Increase value by 1", - "decreaseValueBy1": "Decrease value by 1" + "decreaseValueBy1": "Decrease value by 1", + "preview": "Preview" }, "splashScreen": { "title": { @@ -967,6 +968,29 @@ "bucketOptionLabel": "Bucket {{id}} - {{percentage}}%", "bucketNotActive": "Bucket {{id}} is not active." } + }, + "vectorEmbeddingPolicies": { + "vectorEmbeddingTitle": "Vector embedding {{index}}", + "path": "Path", + "dataType": "Data type", + "distanceFunction": "Distance function", + "dimensions": "Dimensions", + "indexType": "Index type", + "quantizationByteSize": "Quantization byte size", + "quantizationByteSizeTooltip": "This is dynamically set by the {{containerName}} if left blank, or it can be set to a fixed number", + "quantizationByteSizeTooltipContainerName": "container", + "quantizationByteSizeTooltipGlobalSecondaryIndexName": "global secondary index", + "quantizerType": "Quantizer type", + "quantizerTypeTooltip": "The quantization method used by the vector index.", + "indexingSearchListSize": "Indexing search list size", + "vectorIndexShardKey": "Vector index shard key", + "addVectorEmbedding": "Add vector embedding", + "pathEmptyError": "Path should not be empty", + "pathDuplicateError": "Path is already defined", + "dimensionRangeError": "Dimension must be greater than 0 and less than or equal 4096", + "dimensionFlatIndexError": "Maximum allowed dimension for flat index is 505", + "quantizationByteSizeRangeError": "Quantization byte size must be greater than 0 and less than or equal to 512", + "indexingSearchListSizeRangeError": "Indexing search list size must be greater than or equal to 25 and less than or equal to 500" } } }