random-mirrors/cosmos-explorer
1
0
Fork 0
mirror of https://github.com/Azure/cosmos-explorer.git synced 2026-05-14 01:07:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Added quantizerType support in vector embedding (#2471)

Browse Source 
* Added quantizerType support in vector embedding

* Fix build issues

* Address PR Comments

* address pr comments
This commit is contained in:
sunghyunkang1111
2026-05-01 12:38:53 -05:00
committed by GitHub
parent 7014981807
commit 4e8410cd66
8 changed files with 143 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Contracts/DataModels.ts
+1
View File
@@ -255,6 +255,7 @@ export interface VectorIndex {
vectorIndexShardKey?: string[];
indexingSearchListSize?: number;
quantizationByteSize?: number;
quantizerType?: "product" | "spherical";
}
export interface FullTextIndex {
src/Explorer/Controls/Settings/SettingsComponent.tsx
+17 -1
View File
@@ -15,6 +15,7 @@ import {
} from "Explorer/Controls/Settings/SettingsSubComponents/ThroughputInputComponents/ThroughputBucketsComponent";
import { useIndexingPolicyStore } from "Explorer/Tabs/QueryTab/ResultsView";
import { useDatabases } from "Explorer/useDatabases";
import { Keys, t } from "Localization";
import { isFabricNative } from "Platform/Fabric/FabricUtil";
import { isVectorSearchEnabled } from "Utils/CapabilityUtils";
import { isRunningOnPublicCloud } from "Utils/CloudUtils";
@@ -44,7 +45,6 @@ import { useCommandBar } from "../../Menus/CommandBar/CommandBarComponentAdapter
import { SettingsTabV2 } from "../../Tabs/SettingsTabV2";
import "./SettingsComponent.less";
import { mongoIndexingPolicyAADError } from "./SettingsRenderUtils";
import { Keys, t } from "Localization";
import {
ConflictResolutionComponent,
ConflictResolutionComponentProps,
@@ -555,6 +555,19 @@ export class SettingsComponent extends React.Component<SettingsComponentProps, S
private onVectorEmbeddingPolicyChange = (newVectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy): void =>
this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy });
private onVectorIndexesChange = (newVectorIndexes: DataModels.VectorIndex[]): void => {
const currentIndexingPolicy: DataModels.IndexingPolicy =
this.state.indexingPolicyContent || ({} as DataModels.IndexingPolicy);
const newIndexingPolicy: DataModels.IndexingPolicy = {
...currentIndexingPolicy,
vectorIndexes: newVectorIndexes,
};
this.setState({
indexingPolicyContent: newIndexingPolicy,
isIndexingPolicyDirty: true,
});
};
private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void =>
this.setState({ fullTextPolicy: newFullTextPolicy });
@@ -1332,6 +1345,9 @@ export class SettingsComponent extends React.Component<SettingsComponentProps, S
onVectorEmbeddingPolicyChange: this.onVectorEmbeddingPolicyChange,
onVectorEmbeddingPolicyDirtyChange: this.onVectorEmbeddingPolicyDirtyChange,
onVectorEmbeddingPolicyValidationChange: this.onVectorEmbeddingPolicyValidationChange,
vectorIndexes: this.state.indexingPolicyContent?.vectorIndexes ?? [],
vectorIndexesBaseline: this.state.indexingPolicyContentBaseline?.vectorIndexes ?? [],
onVectorIndexesChange: this.onVectorIndexesChange,
isVectorSearchEnabled: this.isVectorSearchEnabled,
fullTextPolicy: this.state.fullTextPolicy,
fullTextPolicyBaseline: this.state.fullTextPolicyBaseline,
src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.tsx
+17 -5
View File
@@ -1,5 +1,5 @@
import { DefaultButton, Pivot, PivotItem, Stack } from "@fluentui/react";
import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy } from "Contracts/DataModels";
import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy, VectorIndex } from "Contracts/DataModels";
import {
FullTextPoliciesComponent,
getFullTextLanguageOptions,
@@ -16,6 +16,9 @@ export interface ContainerPolicyComponentProps {
onVectorEmbeddingPolicyChange: (newVectorEmbeddingPolicy: VectorEmbeddingPolicy) => void;
onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void;
onVectorEmbeddingPolicyValidationChange: (isValid: boolean) => void;
vectorIndexes: VectorIndex[];
vectorIndexesBaseline: VectorIndex[];
onVectorIndexesChange: (newVectorIndexes: VectorIndex[]) => void;
isVectorSearchEnabled: boolean;
fullTextPolicy: FullTextPolicy;
fullTextPolicyBaseline: FullTextPolicy;
@@ -33,6 +36,9 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
onVectorEmbeddingPolicyChange,
onVectorEmbeddingPolicyDirtyChange,
onVectorEmbeddingPolicyValidationChange,
vectorIndexes,
vectorIndexesBaseline,
onVectorIndexesChange,
isVectorSearchEnabled,
fullTextPolicy,
fullTextPolicyBaseline,
@@ -78,6 +84,7 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
const checkAndSendVectorEmbeddingPoliciesToSettings = (
newVectorEmbeddings: VectorEmbedding[],
newVectorIndexes: VectorIndex[],
validationPassed: boolean,
): void => {
onVectorEmbeddingPolicyValidationChange(validationPassed);
@@ -86,6 +93,9 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
if (isVectorDirty) {
onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings });
}
if (isDirty(newVectorIndexes ?? [], vectorIndexesBaseline ?? [])) {
onVectorIndexesChange(newVectorIndexes);
}
};
const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => {
@@ -169,12 +179,14 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
<VectorEmbeddingPoliciesComponent
vectorEmbeddingsBaseline={vectorEmbeddingsBaseline}
vectorEmbeddings={vectorEmbeddings}
vectorIndexes={undefined}
vectorIndexes={vectorIndexes ?? []}
onVectorEmbeddingChange={(
vectorEmbeddings: VectorEmbedding[],
_vectorIndexingPolicies,
newVectorEmbeddings: VectorEmbedding[],
newVectorIndexes: VectorIndex[],
validationPassed: boolean,
) => checkAndSendVectorEmbeddingPoliciesToSettings(vectorEmbeddings, validationPassed)}
) =>
checkAndSendVectorEmbeddingPoliciesToSettings(newVectorEmbeddings, newVectorIndexes, validationPassed)
}
discardChanges={discardVectorChanges}
onChangesDiscarded={onVectorChangesDiscarded}
/>
src/Explorer/Controls/Settings/SettingsUtils.tsx
+2 -1
View File
@@ -1,7 +1,7 @@
import { Keys, t } from "Localization";
import * as Constants from "../../../Common/Constants";
import * as DataModels from "../../../Contracts/DataModels";
import * as ViewModels from "../../../Contracts/ViewModels";
import { Keys, t } from "Localization";
import { isFabricNative } from "../../../Platform/Fabric/FabricUtil";
import { userContext } from "../../../UserContext";
import { isCapabilityEnabled } from "../../../Utils/CapabilityUtils";
@@ -15,6 +15,7 @@ export type isDirtyTypes =
| DataModels.IndexingPolicy
| DataModels.ComputedProperties
| DataModels.VectorEmbedding[]
| DataModels.VectorIndex[]
| DataModels.FullTextPolicy
| DataModels.ThroughputBucket[]
| DataModels.DataMaskingPolicy;
src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap
+3
View File
@@ -359,10 +359,13 @@ exports[`SettingsComponent renders 1`] = `
onVectorEmbeddingPolicyChange={[Function]}
onVectorEmbeddingPolicyDirtyChange={[Function]}
onVectorEmbeddingPolicyValidationChange={[Function]}
onVectorIndexesChange={[Function]}
resetShouldDiscardContainerPolicyChange={[Function]}
shouldDiscardContainerPolicies={false}
vectorEmbeddingPolicy={{}}
vectorEmbeddingPolicyBaseline={{}}
vectorIndexes={[]}
vectorIndexesBaseline={[]}
/>
</Stack>
</PivotItem>
src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx
+69 -23
View File
@@ -16,7 +16,10 @@ import {
getDataTypeOptions,
getDistanceFunctionOptions,
getIndexTypeOptions,
getQuantizerTypeOptions,
supportsQuantization,
} from "Explorer/Controls/VectorSearch/VectorSearchUtils";
import { Keys, t } from "Localization";
import React, { FunctionComponent, useState } from "react";
export interface IVectorEmbeddingPoliciesComponentProps {
@@ -46,6 +49,7 @@ export interface VectorEmbeddingPolicyData {
indexingSearchListSizeError?: string;
quantizationByteSize?: number;
quantizationByteSizeError?: string;
quantizerType?: VectorIndex["quantizerType"];
}
type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType";
@@ -110,7 +114,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onVectorEmbeddingPathError = (path: string, index?: number): string => {
let error = "";
if (!path) {
error = "Path should not be empty";
error = t(Keys.controls.vectorEmbeddingPolicies.pathEmptyError);
}
if (
index >= 0 &&
@@ -119,7 +123,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
dataIndex !== index && vectorEmbedding.path === path,
)
) {
error = "Path is already defined";
error = t(Keys.controls.vectorEmbeddingPolicies.pathDuplicateError);
}
return error;
};
@@ -127,10 +131,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => {
let error = "";
if (dimension <= 0 || dimension > 4096) {
error = "Dimension must be greater than 0 and less than or equal 4096";
error = t(Keys.controls.vectorEmbeddingPolicies.dimensionRangeError);
}
if (indexType === "flat" && dimension > 505) {
error = "Maximum allowed dimension for flat index is 505";
error = t(Keys.controls.vectorEmbeddingPolicies.dimensionFlatIndexError);
}
return error;
};
@@ -138,7 +142,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onQuantizationByteSizeError = (size: number): string => {
let error = "";
if (size < 1 || size > 512) {
error = "Quantization byte size must be greater than 0 and less than or equal to 512";
error = t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeRangeError);
}
return error;
};
@@ -146,7 +150,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onIndexingSearchListSizeError = (size: number): string => {
let error = "";
if (size < 25 || size > 500) {
error = "Indexing search list size must be greater than or equal to 25 and less than or equal to 500";
error = t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSizeRangeError);
}
return error;
};
@@ -155,11 +159,14 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const mergedData: VectorEmbeddingPolicyData[] = [];
vectorEmbeddings?.forEach((embedding) => {
const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined;
const matchingType = matchingIndex?.type;
const supportsQuantizer = supportsQuantization(matchingType);
mergedData.push({
...embedding,
indexType: matchingIndex?.type || "none",
indexType: matchingType || "none",
indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined,
quantizationByteSize: matchingIndex?.quantizationByteSize || undefined,
quantizerType: supportsQuantizer ? matchingIndex?.quantizerType || "product" : undefined,
vectorIndexShardKey: matchingIndex?.vectorIndexShardKey || undefined,
pathError: onVectorEmbeddingPathError(embedding.path),
dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"),
@@ -202,6 +209,9 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
indexingSearchListSize: policy.indexingSearchListSize,
quantizationByteSize: policy.quantizationByteSize,
vectorIndexShardKey: policy.vectorIndexShardKey,
...(supportsQuantization(policy.indexType) && policy.quantizerType
? { quantizerType: policy.quantizerType }
: {}),
}) as VectorIndex,
);
const validationPassed = vectorEmbeddingPolicyData.every(
@@ -245,6 +255,17 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
} else {
vectorEmbedding.indexingSearchListSize = undefined;
}
if (supportsQuantization(vectorEmbedding.indexType)) {
vectorEmbedding.quantizerType = vectorEmbedding.quantizerType || "product";
} else {
vectorEmbedding.quantizerType = undefined;
}
setVectorEmbeddingPolicyData(vectorEmbeddings);
};
const onQuantizerTypeChange = (index: number, option: IDropdownOption): void => {
const vectorEmbeddings = [...vectorEmbeddingPolicyData];
vectorEmbeddings[index].quantizerType = option.key as VectorIndex["quantizerType"];
setVectorEmbeddingPolicyData(vectorEmbeddings);
};
@@ -306,8 +327,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
};
const getQuantizationByteSizeTooltipContent = (): string => {
const containerName: string = isGlobalSecondaryIndex ? "global secondary index" : "container";
return `This is dynamically set by the ${containerName} if left blank, or it can be set to a fixed number`;
const containerName = isGlobalSecondaryIndex
? t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipGlobalSecondaryIndexName)
: t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipContainerName);
return t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltip, { containerName });
};
return (
@@ -319,7 +342,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
key={index}
isExpandedByDefault={true}
title={`Vector embedding ${index + 1}`}
title={t(Keys.controls.vectorEmbeddingPolicies.vectorEmbeddingTitle, { index: index + 1 })}
showDelete={true}
onDelete={() => onDelete(index)}
disableDelete={false}
@@ -337,7 +360,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
>
<Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Path
{t(Keys.controls.vectorEmbeddingPolicies.path)}
</Label>
<TextField
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -352,7 +375,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</Stack>
<Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Data type
{t(Keys.controls.vectorEmbeddingPolicies.dataType)}
</Label>
<Dropdown
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -367,7 +390,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</Stack>
<Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Distance function
{t(Keys.controls.vectorEmbeddingPolicies.distanceFunction)}
</Label>
<Dropdown
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -382,7 +405,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</Stack>
<Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Dimensions
{t(Keys.controls.vectorEmbeddingPolicies.dimensions)}
</Label>
<TextField
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -399,7 +422,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
{displayIndexes && (
<Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Index type
{t(Keys.controls.vectorEmbeddingPolicies.indexType)}
</Label>
<Dropdown
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -415,19 +438,17 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
<Label
disabled={
isExistingPolicy(vectorEmbeddingPolicy) ||
(vectorEmbeddingPolicy.indexType !== "quantizedFlat" &&
vectorEmbeddingPolicy.indexType !== "diskANN")
!supportsQuantization(vectorEmbeddingPolicy.indexType)
}
styles={labelStyles}
>
Quantization byte size
{t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSize)}
<InfoTooltip>{getQuantizationByteSizeTooltipContent()}</InfoTooltip>
</Label>
<TextField
disabled={
isExistingPolicy(vectorEmbeddingPolicy) ||
(vectorEmbeddingPolicy.indexType !== "quantizedFlat" &&
vectorEmbeddingPolicy.indexType !== "diskANN")
!supportsQuantization(vectorEmbeddingPolicy.indexType)
}
id={`vector-policy-quantizationByteSize-${index + 1}`}
styles={textFieldStyles}
@@ -437,6 +458,31 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
}
/>
</Stack>
<Stack style={{ marginLeft: "10px" }}>
<Label
disabled={
isExistingPolicy(vectorEmbeddingPolicy) ||
!supportsQuantization(vectorEmbeddingPolicy.indexType)
}
styles={labelStyles}
>
{t(Keys.controls.vectorEmbeddingPolicies.quantizerType)}
<InfoTooltip>{t(Keys.controls.vectorEmbeddingPolicies.quantizerTypeTooltip)}</InfoTooltip>
</Label>
<Dropdown
disabled={
isExistingPolicy(vectorEmbeddingPolicy) ||
!supportsQuantization(vectorEmbeddingPolicy.indexType)
}
id={`vector-policy-quantizerType-${index + 1}`}
styles={dropdownStyles}
options={getQuantizerTypeOptions()}
selectedKey={vectorEmbeddingPolicy.quantizerType ?? null}
onChange={(_event: React.FormEvent<HTMLDivElement>, option: IDropdownOption) =>
onQuantizerTypeChange(index, option)
}
/>
</Stack>
<Stack style={{ marginLeft: "10px" }}>
<Label
disabled={
@@ -444,7 +490,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
}
styles={labelStyles}
>
Indexing search list size
{t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSize)}
</Label>
<TextField
disabled={
@@ -465,7 +511,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
}
styles={labelStyles}
>
Vector index shard key
{t(Keys.controls.vectorEmbeddingPolicies.vectorIndexShardKey)}
</Label>
<TextField
disabled={
@@ -484,7 +530,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</CollapsibleSectionComponent>
))}
<DefaultButton id={`add-vector-policy`} styles={{ root: { maxWidth: 170, fontSize: 12 } }} onClick={onAdd}>
Add vector embedding
{t(Keys.controls.vectorEmbeddingPolicies.addVectorEmbedding)}
</DefaultButton>
</Stack>
);
src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts
+9
View File
@@ -1,4 +1,6 @@
import { IDropdownOption } from "@fluentui/react";
import { VectorIndex } from "Contracts/DataModels";
import { Keys, t } from "Localization";
const dataTypes = ["float32", "uint8", "int8", "float16"];
const distanceFunctions = ["euclidean", "cosine", "dotproduct"];
@@ -7,6 +9,13 @@ const indexTypes = ["none", "flat", "diskANN", "quantizedFlat"];
export const getDataTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(dataTypes);
export const getDistanceFunctionOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(distanceFunctions);
export const getIndexTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(indexTypes);
export const getQuantizerTypeOptions = (): IDropdownOption[] => [
{ key: "product", text: "Product" },
{ key: "spherical", text: `Spherical (${t(Keys.common.preview)})` },
];
export const supportsQuantization = (indexType: VectorIndex["type"] | "none" | undefined): boolean =>
indexType === "quantizedFlat" || indexType === "diskANN";
function createDropdownOptionsFromLiterals<T extends string>(literals: T[]): IDropdownOption[] {
return literals.map((value) => ({
src/Localization/en/Resources.json
+25 -1
View File
@@ -33,7 +33,8 @@
"publish": "Publish",
"browse": "Browse",
"increaseValueBy1": "Increase value by 1",
"decreaseValueBy1": "Decrease value by 1"
"decreaseValueBy1": "Decrease value by 1",
"preview": "Preview"
},
"splashScreen": {
"title": {
@@ -967,6 +968,29 @@
"bucketOptionLabel": "Bucket {{id}} - {{percentage}}%",
"bucketNotActive": "Bucket {{id}} is not active."
}
},
"vectorEmbeddingPolicies": {
"vectorEmbeddingTitle": "Vector embedding {{index}}",
"path": "Path",
"dataType": "Data type",
"distanceFunction": "Distance function",
"dimensions": "Dimensions",
"indexType": "Index type",
"quantizationByteSize": "Quantization byte size",
"quantizationByteSizeTooltip": "This is dynamically set by the {{containerName}} if left blank, or it can be set to a fixed number",
"quantizationByteSizeTooltipContainerName": "container",
"quantizationByteSizeTooltipGlobalSecondaryIndexName": "global secondary index",
"quantizerType": "Quantizer type",
"quantizerTypeTooltip": "The quantization method used by the vector index.",
"indexingSearchListSize": "Indexing search list size",
"vectorIndexShardKey": "Vector index shard key",
"addVectorEmbedding": "Add vector embedding",
"pathEmptyError": "Path should not be empty",
"pathDuplicateError": "Path is already defined",
"dimensionRangeError": "Dimension must be greater than 0 and less than or equal 4096",
"dimensionFlatIndexError": "Maximum allowed dimension for flat index is 505",
"quantizationByteSizeRangeError": "Quantization byte size must be greater than 0 and less than or equal to 512",
"indexingSearchListSizeRangeError": "Indexing search list size must be greater than or equal to 25 and less than or equal to 500"
}
}
}