Added quantizerType support in vector embedding (#2471)

* Added quantizerType support in vector embedding

* Fix build issues

* Address PR Comments

* address pr comments
This commit is contained in:
sunghyunkang1111
2026-05-01 12:38:53 -05:00
committed by GitHub
parent 7014981807
commit 4e8410cd66
8 changed files with 143 additions and 31 deletions
+1
View File
@@ -255,6 +255,7 @@ export interface VectorIndex {
vectorIndexShardKey?: string[]; vectorIndexShardKey?: string[];
indexingSearchListSize?: number; indexingSearchListSize?: number;
quantizationByteSize?: number; quantizationByteSize?: number;
quantizerType?: "product" | "spherical";
} }
export interface FullTextIndex { export interface FullTextIndex {
@@ -15,6 +15,7 @@ import {
} from "Explorer/Controls/Settings/SettingsSubComponents/ThroughputInputComponents/ThroughputBucketsComponent"; } from "Explorer/Controls/Settings/SettingsSubComponents/ThroughputInputComponents/ThroughputBucketsComponent";
import { useIndexingPolicyStore } from "Explorer/Tabs/QueryTab/ResultsView"; import { useIndexingPolicyStore } from "Explorer/Tabs/QueryTab/ResultsView";
import { useDatabases } from "Explorer/useDatabases"; import { useDatabases } from "Explorer/useDatabases";
import { Keys, t } from "Localization";
import { isFabricNative } from "Platform/Fabric/FabricUtil"; import { isFabricNative } from "Platform/Fabric/FabricUtil";
import { isVectorSearchEnabled } from "Utils/CapabilityUtils"; import { isVectorSearchEnabled } from "Utils/CapabilityUtils";
import { isRunningOnPublicCloud } from "Utils/CloudUtils"; import { isRunningOnPublicCloud } from "Utils/CloudUtils";
@@ -44,7 +45,6 @@ import { useCommandBar } from "../../Menus/CommandBar/CommandBarComponentAdapter
import { SettingsTabV2 } from "../../Tabs/SettingsTabV2"; import { SettingsTabV2 } from "../../Tabs/SettingsTabV2";
import "./SettingsComponent.less"; import "./SettingsComponent.less";
import { mongoIndexingPolicyAADError } from "./SettingsRenderUtils"; import { mongoIndexingPolicyAADError } from "./SettingsRenderUtils";
import { Keys, t } from "Localization";
import { import {
ConflictResolutionComponent, ConflictResolutionComponent,
ConflictResolutionComponentProps, ConflictResolutionComponentProps,
@@ -555,6 +555,19 @@ export class SettingsComponent extends React.Component<SettingsComponentProps, S
private onVectorEmbeddingPolicyChange = (newVectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy): void => private onVectorEmbeddingPolicyChange = (newVectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy): void =>
this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy }); this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy });
private onVectorIndexesChange = (newVectorIndexes: DataModels.VectorIndex[]): void => {
const currentIndexingPolicy: DataModels.IndexingPolicy =
this.state.indexingPolicyContent || ({} as DataModels.IndexingPolicy);
const newIndexingPolicy: DataModels.IndexingPolicy = {
...currentIndexingPolicy,
vectorIndexes: newVectorIndexes,
};
this.setState({
indexingPolicyContent: newIndexingPolicy,
isIndexingPolicyDirty: true,
});
};
private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void => private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void =>
this.setState({ fullTextPolicy: newFullTextPolicy }); this.setState({ fullTextPolicy: newFullTextPolicy });
@@ -1332,6 +1345,9 @@ export class SettingsComponent extends React.Component<SettingsComponentProps, S
onVectorEmbeddingPolicyChange: this.onVectorEmbeddingPolicyChange, onVectorEmbeddingPolicyChange: this.onVectorEmbeddingPolicyChange,
onVectorEmbeddingPolicyDirtyChange: this.onVectorEmbeddingPolicyDirtyChange, onVectorEmbeddingPolicyDirtyChange: this.onVectorEmbeddingPolicyDirtyChange,
onVectorEmbeddingPolicyValidationChange: this.onVectorEmbeddingPolicyValidationChange, onVectorEmbeddingPolicyValidationChange: this.onVectorEmbeddingPolicyValidationChange,
vectorIndexes: this.state.indexingPolicyContent?.vectorIndexes ?? [],
vectorIndexesBaseline: this.state.indexingPolicyContentBaseline?.vectorIndexes ?? [],
onVectorIndexesChange: this.onVectorIndexesChange,
isVectorSearchEnabled: this.isVectorSearchEnabled, isVectorSearchEnabled: this.isVectorSearchEnabled,
fullTextPolicy: this.state.fullTextPolicy, fullTextPolicy: this.state.fullTextPolicy,
fullTextPolicyBaseline: this.state.fullTextPolicyBaseline, fullTextPolicyBaseline: this.state.fullTextPolicyBaseline,
@@ -1,5 +1,5 @@
import { DefaultButton, Pivot, PivotItem, Stack } from "@fluentui/react"; import { DefaultButton, Pivot, PivotItem, Stack } from "@fluentui/react";
import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy } from "Contracts/DataModels"; import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy, VectorIndex } from "Contracts/DataModels";
import { import {
FullTextPoliciesComponent, FullTextPoliciesComponent,
getFullTextLanguageOptions, getFullTextLanguageOptions,
@@ -16,6 +16,9 @@ export interface ContainerPolicyComponentProps {
onVectorEmbeddingPolicyChange: (newVectorEmbeddingPolicy: VectorEmbeddingPolicy) => void; onVectorEmbeddingPolicyChange: (newVectorEmbeddingPolicy: VectorEmbeddingPolicy) => void;
onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void; onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void;
onVectorEmbeddingPolicyValidationChange: (isValid: boolean) => void; onVectorEmbeddingPolicyValidationChange: (isValid: boolean) => void;
vectorIndexes: VectorIndex[];
vectorIndexesBaseline: VectorIndex[];
onVectorIndexesChange: (newVectorIndexes: VectorIndex[]) => void;
isVectorSearchEnabled: boolean; isVectorSearchEnabled: boolean;
fullTextPolicy: FullTextPolicy; fullTextPolicy: FullTextPolicy;
fullTextPolicyBaseline: FullTextPolicy; fullTextPolicyBaseline: FullTextPolicy;
@@ -33,6 +36,9 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
onVectorEmbeddingPolicyChange, onVectorEmbeddingPolicyChange,
onVectorEmbeddingPolicyDirtyChange, onVectorEmbeddingPolicyDirtyChange,
onVectorEmbeddingPolicyValidationChange, onVectorEmbeddingPolicyValidationChange,
vectorIndexes,
vectorIndexesBaseline,
onVectorIndexesChange,
isVectorSearchEnabled, isVectorSearchEnabled,
fullTextPolicy, fullTextPolicy,
fullTextPolicyBaseline, fullTextPolicyBaseline,
@@ -78,6 +84,7 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
const checkAndSendVectorEmbeddingPoliciesToSettings = ( const checkAndSendVectorEmbeddingPoliciesToSettings = (
newVectorEmbeddings: VectorEmbedding[], newVectorEmbeddings: VectorEmbedding[],
newVectorIndexes: VectorIndex[],
validationPassed: boolean, validationPassed: boolean,
): void => { ): void => {
onVectorEmbeddingPolicyValidationChange(validationPassed); onVectorEmbeddingPolicyValidationChange(validationPassed);
@@ -86,6 +93,9 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
if (isVectorDirty) { if (isVectorDirty) {
onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings }); onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings });
} }
if (isDirty(newVectorIndexes ?? [], vectorIndexesBaseline ?? [])) {
onVectorIndexesChange(newVectorIndexes);
}
}; };
const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => { const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => {
@@ -169,12 +179,14 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
<VectorEmbeddingPoliciesComponent <VectorEmbeddingPoliciesComponent
vectorEmbeddingsBaseline={vectorEmbeddingsBaseline} vectorEmbeddingsBaseline={vectorEmbeddingsBaseline}
vectorEmbeddings={vectorEmbeddings} vectorEmbeddings={vectorEmbeddings}
vectorIndexes={undefined} vectorIndexes={vectorIndexes ?? []}
onVectorEmbeddingChange={( onVectorEmbeddingChange={(
vectorEmbeddings: VectorEmbedding[], newVectorEmbeddings: VectorEmbedding[],
_vectorIndexingPolicies, newVectorIndexes: VectorIndex[],
validationPassed: boolean, validationPassed: boolean,
) => checkAndSendVectorEmbeddingPoliciesToSettings(vectorEmbeddings, validationPassed)} ) =>
checkAndSendVectorEmbeddingPoliciesToSettings(newVectorEmbeddings, newVectorIndexes, validationPassed)
}
discardChanges={discardVectorChanges} discardChanges={discardVectorChanges}
onChangesDiscarded={onVectorChangesDiscarded} onChangesDiscarded={onVectorChangesDiscarded}
/> />
@@ -1,7 +1,7 @@
import { Keys, t } from "Localization";
import * as Constants from "../../../Common/Constants"; import * as Constants from "../../../Common/Constants";
import * as DataModels from "../../../Contracts/DataModels"; import * as DataModels from "../../../Contracts/DataModels";
import * as ViewModels from "../../../Contracts/ViewModels"; import * as ViewModels from "../../../Contracts/ViewModels";
import { Keys, t } from "Localization";
import { isFabricNative } from "../../../Platform/Fabric/FabricUtil"; import { isFabricNative } from "../../../Platform/Fabric/FabricUtil";
import { userContext } from "../../../UserContext"; import { userContext } from "../../../UserContext";
import { isCapabilityEnabled } from "../../../Utils/CapabilityUtils"; import { isCapabilityEnabled } from "../../../Utils/CapabilityUtils";
@@ -15,6 +15,7 @@ export type isDirtyTypes =
| DataModels.IndexingPolicy | DataModels.IndexingPolicy
| DataModels.ComputedProperties | DataModels.ComputedProperties
| DataModels.VectorEmbedding[] | DataModels.VectorEmbedding[]
| DataModels.VectorIndex[]
| DataModels.FullTextPolicy | DataModels.FullTextPolicy
| DataModels.ThroughputBucket[] | DataModels.ThroughputBucket[]
| DataModels.DataMaskingPolicy; | DataModels.DataMaskingPolicy;
@@ -359,10 +359,13 @@ exports[`SettingsComponent renders 1`] = `
onVectorEmbeddingPolicyChange={[Function]} onVectorEmbeddingPolicyChange={[Function]}
onVectorEmbeddingPolicyDirtyChange={[Function]} onVectorEmbeddingPolicyDirtyChange={[Function]}
onVectorEmbeddingPolicyValidationChange={[Function]} onVectorEmbeddingPolicyValidationChange={[Function]}
onVectorIndexesChange={[Function]}
resetShouldDiscardContainerPolicyChange={[Function]} resetShouldDiscardContainerPolicyChange={[Function]}
shouldDiscardContainerPolicies={false} shouldDiscardContainerPolicies={false}
vectorEmbeddingPolicy={{}} vectorEmbeddingPolicy={{}}
vectorEmbeddingPolicyBaseline={{}} vectorEmbeddingPolicyBaseline={{}}
vectorIndexes={[]}
vectorIndexesBaseline={[]}
/> />
</Stack> </Stack>
</PivotItem> </PivotItem>
@@ -16,7 +16,10 @@ import {
getDataTypeOptions, getDataTypeOptions,
getDistanceFunctionOptions, getDistanceFunctionOptions,
getIndexTypeOptions, getIndexTypeOptions,
getQuantizerTypeOptions,
supportsQuantization,
} from "Explorer/Controls/VectorSearch/VectorSearchUtils"; } from "Explorer/Controls/VectorSearch/VectorSearchUtils";
import { Keys, t } from "Localization";
import React, { FunctionComponent, useState } from "react"; import React, { FunctionComponent, useState } from "react";
export interface IVectorEmbeddingPoliciesComponentProps { export interface IVectorEmbeddingPoliciesComponentProps {
@@ -46,6 +49,7 @@ export interface VectorEmbeddingPolicyData {
indexingSearchListSizeError?: string; indexingSearchListSizeError?: string;
quantizationByteSize?: number; quantizationByteSize?: number;
quantizationByteSizeError?: string; quantizationByteSizeError?: string;
quantizerType?: VectorIndex["quantizerType"];
} }
type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType"; type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType";
@@ -110,7 +114,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onVectorEmbeddingPathError = (path: string, index?: number): string => { const onVectorEmbeddingPathError = (path: string, index?: number): string => {
let error = ""; let error = "";
if (!path) { if (!path) {
error = "Path should not be empty"; error = t(Keys.controls.vectorEmbeddingPolicies.pathEmptyError);
} }
if ( if (
index >= 0 && index >= 0 &&
@@ -119,7 +123,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
dataIndex !== index && vectorEmbedding.path === path, dataIndex !== index && vectorEmbedding.path === path,
) )
) { ) {
error = "Path is already defined"; error = t(Keys.controls.vectorEmbeddingPolicies.pathDuplicateError);
} }
return error; return error;
}; };
@@ -127,10 +131,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => { const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => {
let error = ""; let error = "";
if (dimension <= 0 || dimension > 4096) { if (dimension <= 0 || dimension > 4096) {
error = "Dimension must be greater than 0 and less than or equal 4096"; error = t(Keys.controls.vectorEmbeddingPolicies.dimensionRangeError);
} }
if (indexType === "flat" && dimension > 505) { if (indexType === "flat" && dimension > 505) {
error = "Maximum allowed dimension for flat index is 505"; error = t(Keys.controls.vectorEmbeddingPolicies.dimensionFlatIndexError);
} }
return error; return error;
}; };
@@ -138,7 +142,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onQuantizationByteSizeError = (size: number): string => { const onQuantizationByteSizeError = (size: number): string => {
let error = ""; let error = "";
if (size < 1 || size > 512) { if (size < 1 || size > 512) {
error = "Quantization byte size must be greater than 0 and less than or equal to 512"; error = t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeRangeError);
} }
return error; return error;
}; };
@@ -146,7 +150,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const onIndexingSearchListSizeError = (size: number): string => { const onIndexingSearchListSizeError = (size: number): string => {
let error = ""; let error = "";
if (size < 25 || size > 500) { if (size < 25 || size > 500) {
error = "Indexing search list size must be greater than or equal to 25 and less than or equal to 500"; error = t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSizeRangeError);
} }
return error; return error;
}; };
@@ -155,11 +159,14 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
const mergedData: VectorEmbeddingPolicyData[] = []; const mergedData: VectorEmbeddingPolicyData[] = [];
vectorEmbeddings?.forEach((embedding) => { vectorEmbeddings?.forEach((embedding) => {
const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined; const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined;
const matchingType = matchingIndex?.type;
const supportsQuantizer = supportsQuantization(matchingType);
mergedData.push({ mergedData.push({
...embedding, ...embedding,
indexType: matchingIndex?.type || "none", indexType: matchingType || "none",
indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined, indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined,
quantizationByteSize: matchingIndex?.quantizationByteSize || undefined, quantizationByteSize: matchingIndex?.quantizationByteSize || undefined,
quantizerType: supportsQuantizer ? matchingIndex?.quantizerType || "product" : undefined,
vectorIndexShardKey: matchingIndex?.vectorIndexShardKey || undefined, vectorIndexShardKey: matchingIndex?.vectorIndexShardKey || undefined,
pathError: onVectorEmbeddingPathError(embedding.path), pathError: onVectorEmbeddingPathError(embedding.path),
dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"), dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"),
@@ -202,6 +209,9 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
indexingSearchListSize: policy.indexingSearchListSize, indexingSearchListSize: policy.indexingSearchListSize,
quantizationByteSize: policy.quantizationByteSize, quantizationByteSize: policy.quantizationByteSize,
vectorIndexShardKey: policy.vectorIndexShardKey, vectorIndexShardKey: policy.vectorIndexShardKey,
...(supportsQuantization(policy.indexType) && policy.quantizerType
? { quantizerType: policy.quantizerType }
: {}),
}) as VectorIndex, }) as VectorIndex,
); );
const validationPassed = vectorEmbeddingPolicyData.every( const validationPassed = vectorEmbeddingPolicyData.every(
@@ -245,6 +255,17 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
} else { } else {
vectorEmbedding.indexingSearchListSize = undefined; vectorEmbedding.indexingSearchListSize = undefined;
} }
if (supportsQuantization(vectorEmbedding.indexType)) {
vectorEmbedding.quantizerType = vectorEmbedding.quantizerType || "product";
} else {
vectorEmbedding.quantizerType = undefined;
}
setVectorEmbeddingPolicyData(vectorEmbeddings);
};
const onQuantizerTypeChange = (index: number, option: IDropdownOption): void => {
const vectorEmbeddings = [...vectorEmbeddingPolicyData];
vectorEmbeddings[index].quantizerType = option.key as VectorIndex["quantizerType"];
setVectorEmbeddingPolicyData(vectorEmbeddings); setVectorEmbeddingPolicyData(vectorEmbeddings);
}; };
@@ -306,8 +327,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
}; };
const getQuantizationByteSizeTooltipContent = (): string => { const getQuantizationByteSizeTooltipContent = (): string => {
const containerName: string = isGlobalSecondaryIndex ? "global secondary index" : "container"; const containerName = isGlobalSecondaryIndex
return `This is dynamically set by the ${containerName} if left blank, or it can be set to a fixed number`; ? t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipGlobalSecondaryIndexName)
: t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipContainerName);
return t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltip, { containerName });
}; };
return ( return (
@@ -319,7 +342,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
disabled={isExistingPolicy(vectorEmbeddingPolicy)} disabled={isExistingPolicy(vectorEmbeddingPolicy)}
key={index} key={index}
isExpandedByDefault={true} isExpandedByDefault={true}
title={`Vector embedding ${index + 1}`} title={t(Keys.controls.vectorEmbeddingPolicies.vectorEmbeddingTitle, { index: index + 1 })}
showDelete={true} showDelete={true}
onDelete={() => onDelete(index)} onDelete={() => onDelete(index)}
disableDelete={false} disableDelete={false}
@@ -337,7 +360,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
> >
<Stack> <Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}> <Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Path {t(Keys.controls.vectorEmbeddingPolicies.path)}
</Label> </Label>
<TextField <TextField
disabled={isExistingPolicy(vectorEmbeddingPolicy)} disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -352,7 +375,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</Stack> </Stack>
<Stack> <Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}> <Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Data type {t(Keys.controls.vectorEmbeddingPolicies.dataType)}
</Label> </Label>
<Dropdown <Dropdown
disabled={isExistingPolicy(vectorEmbeddingPolicy)} disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -367,7 +390,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</Stack> </Stack>
<Stack> <Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}> <Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Distance function {t(Keys.controls.vectorEmbeddingPolicies.distanceFunction)}
</Label> </Label>
<Dropdown <Dropdown
disabled={isExistingPolicy(vectorEmbeddingPolicy)} disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -382,7 +405,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</Stack> </Stack>
<Stack> <Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}> <Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Dimensions {t(Keys.controls.vectorEmbeddingPolicies.dimensions)}
</Label> </Label>
<TextField <TextField
disabled={isExistingPolicy(vectorEmbeddingPolicy)} disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -399,7 +422,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
{displayIndexes && ( {displayIndexes && (
<Stack> <Stack>
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}> <Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
Index type {t(Keys.controls.vectorEmbeddingPolicies.indexType)}
</Label> </Label>
<Dropdown <Dropdown
disabled={isExistingPolicy(vectorEmbeddingPolicy)} disabled={isExistingPolicy(vectorEmbeddingPolicy)}
@@ -415,19 +438,17 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
<Label <Label
disabled={ disabled={
isExistingPolicy(vectorEmbeddingPolicy) || isExistingPolicy(vectorEmbeddingPolicy) ||
(vectorEmbeddingPolicy.indexType !== "quantizedFlat" && !supportsQuantization(vectorEmbeddingPolicy.indexType)
vectorEmbeddingPolicy.indexType !== "diskANN")
} }
styles={labelStyles} styles={labelStyles}
> >
Quantization byte size {t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSize)}
<InfoTooltip>{getQuantizationByteSizeTooltipContent()}</InfoTooltip> <InfoTooltip>{getQuantizationByteSizeTooltipContent()}</InfoTooltip>
</Label> </Label>
<TextField <TextField
disabled={ disabled={
isExistingPolicy(vectorEmbeddingPolicy) || isExistingPolicy(vectorEmbeddingPolicy) ||
(vectorEmbeddingPolicy.indexType !== "quantizedFlat" && !supportsQuantization(vectorEmbeddingPolicy.indexType)
vectorEmbeddingPolicy.indexType !== "diskANN")
} }
id={`vector-policy-quantizationByteSize-${index + 1}`} id={`vector-policy-quantizationByteSize-${index + 1}`}
styles={textFieldStyles} styles={textFieldStyles}
@@ -437,6 +458,31 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
} }
/> />
</Stack> </Stack>
<Stack style={{ marginLeft: "10px" }}>
<Label
disabled={
isExistingPolicy(vectorEmbeddingPolicy) ||
!supportsQuantization(vectorEmbeddingPolicy.indexType)
}
styles={labelStyles}
>
{t(Keys.controls.vectorEmbeddingPolicies.quantizerType)}
<InfoTooltip>{t(Keys.controls.vectorEmbeddingPolicies.quantizerTypeTooltip)}</InfoTooltip>
</Label>
<Dropdown
disabled={
isExistingPolicy(vectorEmbeddingPolicy) ||
!supportsQuantization(vectorEmbeddingPolicy.indexType)
}
id={`vector-policy-quantizerType-${index + 1}`}
styles={dropdownStyles}
options={getQuantizerTypeOptions()}
selectedKey={vectorEmbeddingPolicy.quantizerType ?? null}
onChange={(_event: React.FormEvent<HTMLDivElement>, option: IDropdownOption) =>
onQuantizerTypeChange(index, option)
}
/>
</Stack>
<Stack style={{ marginLeft: "10px" }}> <Stack style={{ marginLeft: "10px" }}>
<Label <Label
disabled={ disabled={
@@ -444,7 +490,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
} }
styles={labelStyles} styles={labelStyles}
> >
Indexing search list size {t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSize)}
</Label> </Label>
<TextField <TextField
disabled={ disabled={
@@ -465,7 +511,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
} }
styles={labelStyles} styles={labelStyles}
> >
Vector index shard key {t(Keys.controls.vectorEmbeddingPolicies.vectorIndexShardKey)}
</Label> </Label>
<TextField <TextField
disabled={ disabled={
@@ -484,7 +530,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
</CollapsibleSectionComponent> </CollapsibleSectionComponent>
))} ))}
<DefaultButton id={`add-vector-policy`} styles={{ root: { maxWidth: 170, fontSize: 12 } }} onClick={onAdd}> <DefaultButton id={`add-vector-policy`} styles={{ root: { maxWidth: 170, fontSize: 12 } }} onClick={onAdd}>
Add vector embedding {t(Keys.controls.vectorEmbeddingPolicies.addVectorEmbedding)}
</DefaultButton> </DefaultButton>
</Stack> </Stack>
); );
@@ -1,4 +1,6 @@
import { IDropdownOption } from "@fluentui/react"; import { IDropdownOption } from "@fluentui/react";
import { VectorIndex } from "Contracts/DataModels";
import { Keys, t } from "Localization";
const dataTypes = ["float32", "uint8", "int8", "float16"]; const dataTypes = ["float32", "uint8", "int8", "float16"];
const distanceFunctions = ["euclidean", "cosine", "dotproduct"]; const distanceFunctions = ["euclidean", "cosine", "dotproduct"];
@@ -7,6 +9,13 @@ const indexTypes = ["none", "flat", "diskANN", "quantizedFlat"];
export const getDataTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(dataTypes); export const getDataTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(dataTypes);
export const getDistanceFunctionOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(distanceFunctions); export const getDistanceFunctionOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(distanceFunctions);
export const getIndexTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(indexTypes); export const getIndexTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(indexTypes);
export const getQuantizerTypeOptions = (): IDropdownOption[] => [
{ key: "product", text: "Product" },
{ key: "spherical", text: `Spherical (${t(Keys.common.preview)})` },
];
export const supportsQuantization = (indexType: VectorIndex["type"] | "none" | undefined): boolean =>
indexType === "quantizedFlat" || indexType === "diskANN";
function createDropdownOptionsFromLiterals<T extends string>(literals: T[]): IDropdownOption[] { function createDropdownOptionsFromLiterals<T extends string>(literals: T[]): IDropdownOption[] {
return literals.map((value) => ({ return literals.map((value) => ({
+25 -1
View File
@@ -33,7 +33,8 @@
"publish": "Publish", "publish": "Publish",
"browse": "Browse", "browse": "Browse",
"increaseValueBy1": "Increase value by 1", "increaseValueBy1": "Increase value by 1",
"decreaseValueBy1": "Decrease value by 1" "decreaseValueBy1": "Decrease value by 1",
"preview": "Preview"
}, },
"splashScreen": { "splashScreen": {
"title": { "title": {
@@ -967,6 +968,29 @@
"bucketOptionLabel": "Bucket {{id}} - {{percentage}}%", "bucketOptionLabel": "Bucket {{id}} - {{percentage}}%",
"bucketNotActive": "Bucket {{id}} is not active." "bucketNotActive": "Bucket {{id}} is not active."
} }
},
"vectorEmbeddingPolicies": {
"vectorEmbeddingTitle": "Vector embedding {{index}}",
"path": "Path",
"dataType": "Data type",
"distanceFunction": "Distance function",
"dimensions": "Dimensions",
"indexType": "Index type",
"quantizationByteSize": "Quantization byte size",
"quantizationByteSizeTooltip": "This is dynamically set by the {{containerName}} if left blank, or it can be set to a fixed number",
"quantizationByteSizeTooltipContainerName": "container",
"quantizationByteSizeTooltipGlobalSecondaryIndexName": "global secondary index",
"quantizerType": "Quantizer type",
"quantizerTypeTooltip": "The quantization method used by the vector index.",
"indexingSearchListSize": "Indexing search list size",
"vectorIndexShardKey": "Vector index shard key",
"addVectorEmbedding": "Add vector embedding",
"pathEmptyError": "Path should not be empty",
"pathDuplicateError": "Path is already defined",
"dimensionRangeError": "Dimension must be greater than 0 and less than or equal 4096",
"dimensionFlatIndexError": "Maximum allowed dimension for flat index is 505",
"quantizationByteSizeRangeError": "Quantization byte size must be greater than 0 and less than or equal to 512",
"indexingSearchListSizeRangeError": "Indexing search list size must be greater than or equal to 25 and less than or equal to 500"
} }
} }
} }