mirror of
https://github.com/Azure/cosmos-explorer.git
synced 2026-05-14 17:27:30 +01:00
Added quantizerType support in vector embedding (#2471)
* Added quantizerType support in vector embedding * Fix build issues * Address PR Comments * address pr comments
This commit is contained in:
@@ -255,6 +255,7 @@ export interface VectorIndex {
|
|||||||
vectorIndexShardKey?: string[];
|
vectorIndexShardKey?: string[];
|
||||||
indexingSearchListSize?: number;
|
indexingSearchListSize?: number;
|
||||||
quantizationByteSize?: number;
|
quantizationByteSize?: number;
|
||||||
|
quantizerType?: "product" | "spherical";
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FullTextIndex {
|
export interface FullTextIndex {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
} from "Explorer/Controls/Settings/SettingsSubComponents/ThroughputInputComponents/ThroughputBucketsComponent";
|
} from "Explorer/Controls/Settings/SettingsSubComponents/ThroughputInputComponents/ThroughputBucketsComponent";
|
||||||
import { useIndexingPolicyStore } from "Explorer/Tabs/QueryTab/ResultsView";
|
import { useIndexingPolicyStore } from "Explorer/Tabs/QueryTab/ResultsView";
|
||||||
import { useDatabases } from "Explorer/useDatabases";
|
import { useDatabases } from "Explorer/useDatabases";
|
||||||
|
import { Keys, t } from "Localization";
|
||||||
import { isFabricNative } from "Platform/Fabric/FabricUtil";
|
import { isFabricNative } from "Platform/Fabric/FabricUtil";
|
||||||
import { isVectorSearchEnabled } from "Utils/CapabilityUtils";
|
import { isVectorSearchEnabled } from "Utils/CapabilityUtils";
|
||||||
import { isRunningOnPublicCloud } from "Utils/CloudUtils";
|
import { isRunningOnPublicCloud } from "Utils/CloudUtils";
|
||||||
@@ -44,7 +45,6 @@ import { useCommandBar } from "../../Menus/CommandBar/CommandBarComponentAdapter
|
|||||||
import { SettingsTabV2 } from "../../Tabs/SettingsTabV2";
|
import { SettingsTabV2 } from "../../Tabs/SettingsTabV2";
|
||||||
import "./SettingsComponent.less";
|
import "./SettingsComponent.less";
|
||||||
import { mongoIndexingPolicyAADError } from "./SettingsRenderUtils";
|
import { mongoIndexingPolicyAADError } from "./SettingsRenderUtils";
|
||||||
import { Keys, t } from "Localization";
|
|
||||||
import {
|
import {
|
||||||
ConflictResolutionComponent,
|
ConflictResolutionComponent,
|
||||||
ConflictResolutionComponentProps,
|
ConflictResolutionComponentProps,
|
||||||
@@ -555,6 +555,19 @@ export class SettingsComponent extends React.Component<SettingsComponentProps, S
|
|||||||
private onVectorEmbeddingPolicyChange = (newVectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy): void =>
|
private onVectorEmbeddingPolicyChange = (newVectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy): void =>
|
||||||
this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy });
|
this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy });
|
||||||
|
|
||||||
|
private onVectorIndexesChange = (newVectorIndexes: DataModels.VectorIndex[]): void => {
|
||||||
|
const currentIndexingPolicy: DataModels.IndexingPolicy =
|
||||||
|
this.state.indexingPolicyContent || ({} as DataModels.IndexingPolicy);
|
||||||
|
const newIndexingPolicy: DataModels.IndexingPolicy = {
|
||||||
|
...currentIndexingPolicy,
|
||||||
|
vectorIndexes: newVectorIndexes,
|
||||||
|
};
|
||||||
|
this.setState({
|
||||||
|
indexingPolicyContent: newIndexingPolicy,
|
||||||
|
isIndexingPolicyDirty: true,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void =>
|
private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void =>
|
||||||
this.setState({ fullTextPolicy: newFullTextPolicy });
|
this.setState({ fullTextPolicy: newFullTextPolicy });
|
||||||
|
|
||||||
@@ -1332,6 +1345,9 @@ export class SettingsComponent extends React.Component<SettingsComponentProps, S
|
|||||||
onVectorEmbeddingPolicyChange: this.onVectorEmbeddingPolicyChange,
|
onVectorEmbeddingPolicyChange: this.onVectorEmbeddingPolicyChange,
|
||||||
onVectorEmbeddingPolicyDirtyChange: this.onVectorEmbeddingPolicyDirtyChange,
|
onVectorEmbeddingPolicyDirtyChange: this.onVectorEmbeddingPolicyDirtyChange,
|
||||||
onVectorEmbeddingPolicyValidationChange: this.onVectorEmbeddingPolicyValidationChange,
|
onVectorEmbeddingPolicyValidationChange: this.onVectorEmbeddingPolicyValidationChange,
|
||||||
|
vectorIndexes: this.state.indexingPolicyContent?.vectorIndexes ?? [],
|
||||||
|
vectorIndexesBaseline: this.state.indexingPolicyContentBaseline?.vectorIndexes ?? [],
|
||||||
|
onVectorIndexesChange: this.onVectorIndexesChange,
|
||||||
isVectorSearchEnabled: this.isVectorSearchEnabled,
|
isVectorSearchEnabled: this.isVectorSearchEnabled,
|
||||||
fullTextPolicy: this.state.fullTextPolicy,
|
fullTextPolicy: this.state.fullTextPolicy,
|
||||||
fullTextPolicyBaseline: this.state.fullTextPolicyBaseline,
|
fullTextPolicyBaseline: this.state.fullTextPolicyBaseline,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { DefaultButton, Pivot, PivotItem, Stack } from "@fluentui/react";
|
import { DefaultButton, Pivot, PivotItem, Stack } from "@fluentui/react";
|
||||||
import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy } from "Contracts/DataModels";
|
import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy, VectorIndex } from "Contracts/DataModels";
|
||||||
import {
|
import {
|
||||||
FullTextPoliciesComponent,
|
FullTextPoliciesComponent,
|
||||||
getFullTextLanguageOptions,
|
getFullTextLanguageOptions,
|
||||||
@@ -16,6 +16,9 @@ export interface ContainerPolicyComponentProps {
|
|||||||
onVectorEmbeddingPolicyChange: (newVectorEmbeddingPolicy: VectorEmbeddingPolicy) => void;
|
onVectorEmbeddingPolicyChange: (newVectorEmbeddingPolicy: VectorEmbeddingPolicy) => void;
|
||||||
onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void;
|
onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void;
|
||||||
onVectorEmbeddingPolicyValidationChange: (isValid: boolean) => void;
|
onVectorEmbeddingPolicyValidationChange: (isValid: boolean) => void;
|
||||||
|
vectorIndexes: VectorIndex[];
|
||||||
|
vectorIndexesBaseline: VectorIndex[];
|
||||||
|
onVectorIndexesChange: (newVectorIndexes: VectorIndex[]) => void;
|
||||||
isVectorSearchEnabled: boolean;
|
isVectorSearchEnabled: boolean;
|
||||||
fullTextPolicy: FullTextPolicy;
|
fullTextPolicy: FullTextPolicy;
|
||||||
fullTextPolicyBaseline: FullTextPolicy;
|
fullTextPolicyBaseline: FullTextPolicy;
|
||||||
@@ -33,6 +36,9 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
|
|||||||
onVectorEmbeddingPolicyChange,
|
onVectorEmbeddingPolicyChange,
|
||||||
onVectorEmbeddingPolicyDirtyChange,
|
onVectorEmbeddingPolicyDirtyChange,
|
||||||
onVectorEmbeddingPolicyValidationChange,
|
onVectorEmbeddingPolicyValidationChange,
|
||||||
|
vectorIndexes,
|
||||||
|
vectorIndexesBaseline,
|
||||||
|
onVectorIndexesChange,
|
||||||
isVectorSearchEnabled,
|
isVectorSearchEnabled,
|
||||||
fullTextPolicy,
|
fullTextPolicy,
|
||||||
fullTextPolicyBaseline,
|
fullTextPolicyBaseline,
|
||||||
@@ -78,6 +84,7 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
|
|||||||
|
|
||||||
const checkAndSendVectorEmbeddingPoliciesToSettings = (
|
const checkAndSendVectorEmbeddingPoliciesToSettings = (
|
||||||
newVectorEmbeddings: VectorEmbedding[],
|
newVectorEmbeddings: VectorEmbedding[],
|
||||||
|
newVectorIndexes: VectorIndex[],
|
||||||
validationPassed: boolean,
|
validationPassed: boolean,
|
||||||
): void => {
|
): void => {
|
||||||
onVectorEmbeddingPolicyValidationChange(validationPassed);
|
onVectorEmbeddingPolicyValidationChange(validationPassed);
|
||||||
@@ -86,6 +93,9 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
|
|||||||
if (isVectorDirty) {
|
if (isVectorDirty) {
|
||||||
onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings });
|
onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings });
|
||||||
}
|
}
|
||||||
|
if (isDirty(newVectorIndexes ?? [], vectorIndexesBaseline ?? [])) {
|
||||||
|
onVectorIndexesChange(newVectorIndexes);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => {
|
const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => {
|
||||||
@@ -169,12 +179,14 @@ export const ContainerPolicyComponent: React.FC<ContainerPolicyComponentProps> =
|
|||||||
<VectorEmbeddingPoliciesComponent
|
<VectorEmbeddingPoliciesComponent
|
||||||
vectorEmbeddingsBaseline={vectorEmbeddingsBaseline}
|
vectorEmbeddingsBaseline={vectorEmbeddingsBaseline}
|
||||||
vectorEmbeddings={vectorEmbeddings}
|
vectorEmbeddings={vectorEmbeddings}
|
||||||
vectorIndexes={undefined}
|
vectorIndexes={vectorIndexes ?? []}
|
||||||
onVectorEmbeddingChange={(
|
onVectorEmbeddingChange={(
|
||||||
vectorEmbeddings: VectorEmbedding[],
|
newVectorEmbeddings: VectorEmbedding[],
|
||||||
_vectorIndexingPolicies,
|
newVectorIndexes: VectorIndex[],
|
||||||
validationPassed: boolean,
|
validationPassed: boolean,
|
||||||
) => checkAndSendVectorEmbeddingPoliciesToSettings(vectorEmbeddings, validationPassed)}
|
) =>
|
||||||
|
checkAndSendVectorEmbeddingPoliciesToSettings(newVectorEmbeddings, newVectorIndexes, validationPassed)
|
||||||
|
}
|
||||||
discardChanges={discardVectorChanges}
|
discardChanges={discardVectorChanges}
|
||||||
onChangesDiscarded={onVectorChangesDiscarded}
|
onChangesDiscarded={onVectorChangesDiscarded}
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
|
import { Keys, t } from "Localization";
|
||||||
import * as Constants from "../../../Common/Constants";
|
import * as Constants from "../../../Common/Constants";
|
||||||
import * as DataModels from "../../../Contracts/DataModels";
|
import * as DataModels from "../../../Contracts/DataModels";
|
||||||
import * as ViewModels from "../../../Contracts/ViewModels";
|
import * as ViewModels from "../../../Contracts/ViewModels";
|
||||||
import { Keys, t } from "Localization";
|
|
||||||
import { isFabricNative } from "../../../Platform/Fabric/FabricUtil";
|
import { isFabricNative } from "../../../Platform/Fabric/FabricUtil";
|
||||||
import { userContext } from "../../../UserContext";
|
import { userContext } from "../../../UserContext";
|
||||||
import { isCapabilityEnabled } from "../../../Utils/CapabilityUtils";
|
import { isCapabilityEnabled } from "../../../Utils/CapabilityUtils";
|
||||||
@@ -15,6 +15,7 @@ export type isDirtyTypes =
|
|||||||
| DataModels.IndexingPolicy
|
| DataModels.IndexingPolicy
|
||||||
| DataModels.ComputedProperties
|
| DataModels.ComputedProperties
|
||||||
| DataModels.VectorEmbedding[]
|
| DataModels.VectorEmbedding[]
|
||||||
|
| DataModels.VectorIndex[]
|
||||||
| DataModels.FullTextPolicy
|
| DataModels.FullTextPolicy
|
||||||
| DataModels.ThroughputBucket[]
|
| DataModels.ThroughputBucket[]
|
||||||
| DataModels.DataMaskingPolicy;
|
| DataModels.DataMaskingPolicy;
|
||||||
|
|||||||
@@ -359,10 +359,13 @@ exports[`SettingsComponent renders 1`] = `
|
|||||||
onVectorEmbeddingPolicyChange={[Function]}
|
onVectorEmbeddingPolicyChange={[Function]}
|
||||||
onVectorEmbeddingPolicyDirtyChange={[Function]}
|
onVectorEmbeddingPolicyDirtyChange={[Function]}
|
||||||
onVectorEmbeddingPolicyValidationChange={[Function]}
|
onVectorEmbeddingPolicyValidationChange={[Function]}
|
||||||
|
onVectorIndexesChange={[Function]}
|
||||||
resetShouldDiscardContainerPolicyChange={[Function]}
|
resetShouldDiscardContainerPolicyChange={[Function]}
|
||||||
shouldDiscardContainerPolicies={false}
|
shouldDiscardContainerPolicies={false}
|
||||||
vectorEmbeddingPolicy={{}}
|
vectorEmbeddingPolicy={{}}
|
||||||
vectorEmbeddingPolicyBaseline={{}}
|
vectorEmbeddingPolicyBaseline={{}}
|
||||||
|
vectorIndexes={[]}
|
||||||
|
vectorIndexesBaseline={[]}
|
||||||
/>
|
/>
|
||||||
</Stack>
|
</Stack>
|
||||||
</PivotItem>
|
</PivotItem>
|
||||||
|
|||||||
@@ -16,7 +16,10 @@ import {
|
|||||||
getDataTypeOptions,
|
getDataTypeOptions,
|
||||||
getDistanceFunctionOptions,
|
getDistanceFunctionOptions,
|
||||||
getIndexTypeOptions,
|
getIndexTypeOptions,
|
||||||
|
getQuantizerTypeOptions,
|
||||||
|
supportsQuantization,
|
||||||
} from "Explorer/Controls/VectorSearch/VectorSearchUtils";
|
} from "Explorer/Controls/VectorSearch/VectorSearchUtils";
|
||||||
|
import { Keys, t } from "Localization";
|
||||||
import React, { FunctionComponent, useState } from "react";
|
import React, { FunctionComponent, useState } from "react";
|
||||||
|
|
||||||
export interface IVectorEmbeddingPoliciesComponentProps {
|
export interface IVectorEmbeddingPoliciesComponentProps {
|
||||||
@@ -46,6 +49,7 @@ export interface VectorEmbeddingPolicyData {
|
|||||||
indexingSearchListSizeError?: string;
|
indexingSearchListSizeError?: string;
|
||||||
quantizationByteSize?: number;
|
quantizationByteSize?: number;
|
||||||
quantizationByteSizeError?: string;
|
quantizationByteSizeError?: string;
|
||||||
|
quantizerType?: VectorIndex["quantizerType"];
|
||||||
}
|
}
|
||||||
|
|
||||||
type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType";
|
type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType";
|
||||||
@@ -110,7 +114,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
const onVectorEmbeddingPathError = (path: string, index?: number): string => {
|
const onVectorEmbeddingPathError = (path: string, index?: number): string => {
|
||||||
let error = "";
|
let error = "";
|
||||||
if (!path) {
|
if (!path) {
|
||||||
error = "Path should not be empty";
|
error = t(Keys.controls.vectorEmbeddingPolicies.pathEmptyError);
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
index >= 0 &&
|
index >= 0 &&
|
||||||
@@ -119,7 +123,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
dataIndex !== index && vectorEmbedding.path === path,
|
dataIndex !== index && vectorEmbedding.path === path,
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
error = "Path is already defined";
|
error = t(Keys.controls.vectorEmbeddingPolicies.pathDuplicateError);
|
||||||
}
|
}
|
||||||
return error;
|
return error;
|
||||||
};
|
};
|
||||||
@@ -127,10 +131,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => {
|
const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => {
|
||||||
let error = "";
|
let error = "";
|
||||||
if (dimension <= 0 || dimension > 4096) {
|
if (dimension <= 0 || dimension > 4096) {
|
||||||
error = "Dimension must be greater than 0 and less than or equal 4096";
|
error = t(Keys.controls.vectorEmbeddingPolicies.dimensionRangeError);
|
||||||
}
|
}
|
||||||
if (indexType === "flat" && dimension > 505) {
|
if (indexType === "flat" && dimension > 505) {
|
||||||
error = "Maximum allowed dimension for flat index is 505";
|
error = t(Keys.controls.vectorEmbeddingPolicies.dimensionFlatIndexError);
|
||||||
}
|
}
|
||||||
return error;
|
return error;
|
||||||
};
|
};
|
||||||
@@ -138,7 +142,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
const onQuantizationByteSizeError = (size: number): string => {
|
const onQuantizationByteSizeError = (size: number): string => {
|
||||||
let error = "";
|
let error = "";
|
||||||
if (size < 1 || size > 512) {
|
if (size < 1 || size > 512) {
|
||||||
error = "Quantization byte size must be greater than 0 and less than or equal to 512";
|
error = t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeRangeError);
|
||||||
}
|
}
|
||||||
return error;
|
return error;
|
||||||
};
|
};
|
||||||
@@ -146,7 +150,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
const onIndexingSearchListSizeError = (size: number): string => {
|
const onIndexingSearchListSizeError = (size: number): string => {
|
||||||
let error = "";
|
let error = "";
|
||||||
if (size < 25 || size > 500) {
|
if (size < 25 || size > 500) {
|
||||||
error = "Indexing search list size must be greater than or equal to 25 and less than or equal to 500";
|
error = t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSizeRangeError);
|
||||||
}
|
}
|
||||||
return error;
|
return error;
|
||||||
};
|
};
|
||||||
@@ -155,11 +159,14 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
const mergedData: VectorEmbeddingPolicyData[] = [];
|
const mergedData: VectorEmbeddingPolicyData[] = [];
|
||||||
vectorEmbeddings?.forEach((embedding) => {
|
vectorEmbeddings?.forEach((embedding) => {
|
||||||
const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined;
|
const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined;
|
||||||
|
const matchingType = matchingIndex?.type;
|
||||||
|
const supportsQuantizer = supportsQuantization(matchingType);
|
||||||
mergedData.push({
|
mergedData.push({
|
||||||
...embedding,
|
...embedding,
|
||||||
indexType: matchingIndex?.type || "none",
|
indexType: matchingType || "none",
|
||||||
indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined,
|
indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined,
|
||||||
quantizationByteSize: matchingIndex?.quantizationByteSize || undefined,
|
quantizationByteSize: matchingIndex?.quantizationByteSize || undefined,
|
||||||
|
quantizerType: supportsQuantizer ? matchingIndex?.quantizerType || "product" : undefined,
|
||||||
vectorIndexShardKey: matchingIndex?.vectorIndexShardKey || undefined,
|
vectorIndexShardKey: matchingIndex?.vectorIndexShardKey || undefined,
|
||||||
pathError: onVectorEmbeddingPathError(embedding.path),
|
pathError: onVectorEmbeddingPathError(embedding.path),
|
||||||
dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"),
|
dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"),
|
||||||
@@ -202,6 +209,9 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
indexingSearchListSize: policy.indexingSearchListSize,
|
indexingSearchListSize: policy.indexingSearchListSize,
|
||||||
quantizationByteSize: policy.quantizationByteSize,
|
quantizationByteSize: policy.quantizationByteSize,
|
||||||
vectorIndexShardKey: policy.vectorIndexShardKey,
|
vectorIndexShardKey: policy.vectorIndexShardKey,
|
||||||
|
...(supportsQuantization(policy.indexType) && policy.quantizerType
|
||||||
|
? { quantizerType: policy.quantizerType }
|
||||||
|
: {}),
|
||||||
}) as VectorIndex,
|
}) as VectorIndex,
|
||||||
);
|
);
|
||||||
const validationPassed = vectorEmbeddingPolicyData.every(
|
const validationPassed = vectorEmbeddingPolicyData.every(
|
||||||
@@ -245,6 +255,17 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
} else {
|
} else {
|
||||||
vectorEmbedding.indexingSearchListSize = undefined;
|
vectorEmbedding.indexingSearchListSize = undefined;
|
||||||
}
|
}
|
||||||
|
if (supportsQuantization(vectorEmbedding.indexType)) {
|
||||||
|
vectorEmbedding.quantizerType = vectorEmbedding.quantizerType || "product";
|
||||||
|
} else {
|
||||||
|
vectorEmbedding.quantizerType = undefined;
|
||||||
|
}
|
||||||
|
setVectorEmbeddingPolicyData(vectorEmbeddings);
|
||||||
|
};
|
||||||
|
|
||||||
|
const onQuantizerTypeChange = (index: number, option: IDropdownOption): void => {
|
||||||
|
const vectorEmbeddings = [...vectorEmbeddingPolicyData];
|
||||||
|
vectorEmbeddings[index].quantizerType = option.key as VectorIndex["quantizerType"];
|
||||||
setVectorEmbeddingPolicyData(vectorEmbeddings);
|
setVectorEmbeddingPolicyData(vectorEmbeddings);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -306,8 +327,10 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
};
|
};
|
||||||
|
|
||||||
const getQuantizationByteSizeTooltipContent = (): string => {
|
const getQuantizationByteSizeTooltipContent = (): string => {
|
||||||
const containerName: string = isGlobalSecondaryIndex ? "global secondary index" : "container";
|
const containerName = isGlobalSecondaryIndex
|
||||||
return `This is dynamically set by the ${containerName} if left blank, or it can be set to a fixed number`;
|
? t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipGlobalSecondaryIndexName)
|
||||||
|
: t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltipContainerName);
|
||||||
|
return t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSizeTooltip, { containerName });
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -319,7 +342,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
||||||
key={index}
|
key={index}
|
||||||
isExpandedByDefault={true}
|
isExpandedByDefault={true}
|
||||||
title={`Vector embedding ${index + 1}`}
|
title={t(Keys.controls.vectorEmbeddingPolicies.vectorEmbeddingTitle, { index: index + 1 })}
|
||||||
showDelete={true}
|
showDelete={true}
|
||||||
onDelete={() => onDelete(index)}
|
onDelete={() => onDelete(index)}
|
||||||
disableDelete={false}
|
disableDelete={false}
|
||||||
@@ -337,7 +360,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
>
|
>
|
||||||
<Stack>
|
<Stack>
|
||||||
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
||||||
Path
|
{t(Keys.controls.vectorEmbeddingPolicies.path)}
|
||||||
</Label>
|
</Label>
|
||||||
<TextField
|
<TextField
|
||||||
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
||||||
@@ -352,7 +375,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
</Stack>
|
</Stack>
|
||||||
<Stack>
|
<Stack>
|
||||||
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
||||||
Data type
|
{t(Keys.controls.vectorEmbeddingPolicies.dataType)}
|
||||||
</Label>
|
</Label>
|
||||||
<Dropdown
|
<Dropdown
|
||||||
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
||||||
@@ -367,7 +390,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
</Stack>
|
</Stack>
|
||||||
<Stack>
|
<Stack>
|
||||||
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
||||||
Distance function
|
{t(Keys.controls.vectorEmbeddingPolicies.distanceFunction)}
|
||||||
</Label>
|
</Label>
|
||||||
<Dropdown
|
<Dropdown
|
||||||
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
||||||
@@ -382,7 +405,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
</Stack>
|
</Stack>
|
||||||
<Stack>
|
<Stack>
|
||||||
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
||||||
Dimensions
|
{t(Keys.controls.vectorEmbeddingPolicies.dimensions)}
|
||||||
</Label>
|
</Label>
|
||||||
<TextField
|
<TextField
|
||||||
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
||||||
@@ -399,7 +422,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
{displayIndexes && (
|
{displayIndexes && (
|
||||||
<Stack>
|
<Stack>
|
||||||
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
<Label disabled={isExistingPolicy(vectorEmbeddingPolicy)} styles={labelStyles}>
|
||||||
Index type
|
{t(Keys.controls.vectorEmbeddingPolicies.indexType)}
|
||||||
</Label>
|
</Label>
|
||||||
<Dropdown
|
<Dropdown
|
||||||
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
disabled={isExistingPolicy(vectorEmbeddingPolicy)}
|
||||||
@@ -415,19 +438,17 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
<Label
|
<Label
|
||||||
disabled={
|
disabled={
|
||||||
isExistingPolicy(vectorEmbeddingPolicy) ||
|
isExistingPolicy(vectorEmbeddingPolicy) ||
|
||||||
(vectorEmbeddingPolicy.indexType !== "quantizedFlat" &&
|
!supportsQuantization(vectorEmbeddingPolicy.indexType)
|
||||||
vectorEmbeddingPolicy.indexType !== "diskANN")
|
|
||||||
}
|
}
|
||||||
styles={labelStyles}
|
styles={labelStyles}
|
||||||
>
|
>
|
||||||
Quantization byte size
|
{t(Keys.controls.vectorEmbeddingPolicies.quantizationByteSize)}
|
||||||
<InfoTooltip>{getQuantizationByteSizeTooltipContent()}</InfoTooltip>
|
<InfoTooltip>{getQuantizationByteSizeTooltipContent()}</InfoTooltip>
|
||||||
</Label>
|
</Label>
|
||||||
<TextField
|
<TextField
|
||||||
disabled={
|
disabled={
|
||||||
isExistingPolicy(vectorEmbeddingPolicy) ||
|
isExistingPolicy(vectorEmbeddingPolicy) ||
|
||||||
(vectorEmbeddingPolicy.indexType !== "quantizedFlat" &&
|
!supportsQuantization(vectorEmbeddingPolicy.indexType)
|
||||||
vectorEmbeddingPolicy.indexType !== "diskANN")
|
|
||||||
}
|
}
|
||||||
id={`vector-policy-quantizationByteSize-${index + 1}`}
|
id={`vector-policy-quantizationByteSize-${index + 1}`}
|
||||||
styles={textFieldStyles}
|
styles={textFieldStyles}
|
||||||
@@ -437,6 +458,31 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
</Stack>
|
</Stack>
|
||||||
|
<Stack style={{ marginLeft: "10px" }}>
|
||||||
|
<Label
|
||||||
|
disabled={
|
||||||
|
isExistingPolicy(vectorEmbeddingPolicy) ||
|
||||||
|
!supportsQuantization(vectorEmbeddingPolicy.indexType)
|
||||||
|
}
|
||||||
|
styles={labelStyles}
|
||||||
|
>
|
||||||
|
{t(Keys.controls.vectorEmbeddingPolicies.quantizerType)}
|
||||||
|
<InfoTooltip>{t(Keys.controls.vectorEmbeddingPolicies.quantizerTypeTooltip)}</InfoTooltip>
|
||||||
|
</Label>
|
||||||
|
<Dropdown
|
||||||
|
disabled={
|
||||||
|
isExistingPolicy(vectorEmbeddingPolicy) ||
|
||||||
|
!supportsQuantization(vectorEmbeddingPolicy.indexType)
|
||||||
|
}
|
||||||
|
id={`vector-policy-quantizerType-${index + 1}`}
|
||||||
|
styles={dropdownStyles}
|
||||||
|
options={getQuantizerTypeOptions()}
|
||||||
|
selectedKey={vectorEmbeddingPolicy.quantizerType ?? null}
|
||||||
|
onChange={(_event: React.FormEvent<HTMLDivElement>, option: IDropdownOption) =>
|
||||||
|
onQuantizerTypeChange(index, option)
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</Stack>
|
||||||
<Stack style={{ marginLeft: "10px" }}>
|
<Stack style={{ marginLeft: "10px" }}>
|
||||||
<Label
|
<Label
|
||||||
disabled={
|
disabled={
|
||||||
@@ -444,7 +490,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
}
|
}
|
||||||
styles={labelStyles}
|
styles={labelStyles}
|
||||||
>
|
>
|
||||||
Indexing search list size
|
{t(Keys.controls.vectorEmbeddingPolicies.indexingSearchListSize)}
|
||||||
</Label>
|
</Label>
|
||||||
<TextField
|
<TextField
|
||||||
disabled={
|
disabled={
|
||||||
@@ -465,7 +511,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
}
|
}
|
||||||
styles={labelStyles}
|
styles={labelStyles}
|
||||||
>
|
>
|
||||||
Vector index shard key
|
{t(Keys.controls.vectorEmbeddingPolicies.vectorIndexShardKey)}
|
||||||
</Label>
|
</Label>
|
||||||
<TextField
|
<TextField
|
||||||
disabled={
|
disabled={
|
||||||
@@ -484,7 +530,7 @@ export const VectorEmbeddingPoliciesComponent: FunctionComponent<IVectorEmbeddin
|
|||||||
</CollapsibleSectionComponent>
|
</CollapsibleSectionComponent>
|
||||||
))}
|
))}
|
||||||
<DefaultButton id={`add-vector-policy`} styles={{ root: { maxWidth: 170, fontSize: 12 } }} onClick={onAdd}>
|
<DefaultButton id={`add-vector-policy`} styles={{ root: { maxWidth: 170, fontSize: 12 } }} onClick={onAdd}>
|
||||||
Add vector embedding
|
{t(Keys.controls.vectorEmbeddingPolicies.addVectorEmbedding)}
|
||||||
</DefaultButton>
|
</DefaultButton>
|
||||||
</Stack>
|
</Stack>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
import { IDropdownOption } from "@fluentui/react";
|
import { IDropdownOption } from "@fluentui/react";
|
||||||
|
import { VectorIndex } from "Contracts/DataModels";
|
||||||
|
import { Keys, t } from "Localization";
|
||||||
|
|
||||||
const dataTypes = ["float32", "uint8", "int8", "float16"];
|
const dataTypes = ["float32", "uint8", "int8", "float16"];
|
||||||
const distanceFunctions = ["euclidean", "cosine", "dotproduct"];
|
const distanceFunctions = ["euclidean", "cosine", "dotproduct"];
|
||||||
@@ -7,6 +9,13 @@ const indexTypes = ["none", "flat", "diskANN", "quantizedFlat"];
|
|||||||
export const getDataTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(dataTypes);
|
export const getDataTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(dataTypes);
|
||||||
export const getDistanceFunctionOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(distanceFunctions);
|
export const getDistanceFunctionOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(distanceFunctions);
|
||||||
export const getIndexTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(indexTypes);
|
export const getIndexTypeOptions = (): IDropdownOption[] => createDropdownOptionsFromLiterals(indexTypes);
|
||||||
|
export const getQuantizerTypeOptions = (): IDropdownOption[] => [
|
||||||
|
{ key: "product", text: "Product" },
|
||||||
|
{ key: "spherical", text: `Spherical (${t(Keys.common.preview)})` },
|
||||||
|
];
|
||||||
|
|
||||||
|
export const supportsQuantization = (indexType: VectorIndex["type"] | "none" | undefined): boolean =>
|
||||||
|
indexType === "quantizedFlat" || indexType === "diskANN";
|
||||||
|
|
||||||
function createDropdownOptionsFromLiterals<T extends string>(literals: T[]): IDropdownOption[] {
|
function createDropdownOptionsFromLiterals<T extends string>(literals: T[]): IDropdownOption[] {
|
||||||
return literals.map((value) => ({
|
return literals.map((value) => ({
|
||||||
|
|||||||
@@ -33,7 +33,8 @@
|
|||||||
"publish": "Publish",
|
"publish": "Publish",
|
||||||
"browse": "Browse",
|
"browse": "Browse",
|
||||||
"increaseValueBy1": "Increase value by 1",
|
"increaseValueBy1": "Increase value by 1",
|
||||||
"decreaseValueBy1": "Decrease value by 1"
|
"decreaseValueBy1": "Decrease value by 1",
|
||||||
|
"preview": "Preview"
|
||||||
},
|
},
|
||||||
"splashScreen": {
|
"splashScreen": {
|
||||||
"title": {
|
"title": {
|
||||||
@@ -967,6 +968,29 @@
|
|||||||
"bucketOptionLabel": "Bucket {{id}} - {{percentage}}%",
|
"bucketOptionLabel": "Bucket {{id}} - {{percentage}}%",
|
||||||
"bucketNotActive": "Bucket {{id}} is not active."
|
"bucketNotActive": "Bucket {{id}} is not active."
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"vectorEmbeddingPolicies": {
|
||||||
|
"vectorEmbeddingTitle": "Vector embedding {{index}}",
|
||||||
|
"path": "Path",
|
||||||
|
"dataType": "Data type",
|
||||||
|
"distanceFunction": "Distance function",
|
||||||
|
"dimensions": "Dimensions",
|
||||||
|
"indexType": "Index type",
|
||||||
|
"quantizationByteSize": "Quantization byte size",
|
||||||
|
"quantizationByteSizeTooltip": "This is dynamically set by the {{containerName}} if left blank, or it can be set to a fixed number",
|
||||||
|
"quantizationByteSizeTooltipContainerName": "container",
|
||||||
|
"quantizationByteSizeTooltipGlobalSecondaryIndexName": "global secondary index",
|
||||||
|
"quantizerType": "Quantizer type",
|
||||||
|
"quantizerTypeTooltip": "The quantization method used by the vector index.",
|
||||||
|
"indexingSearchListSize": "Indexing search list size",
|
||||||
|
"vectorIndexShardKey": "Vector index shard key",
|
||||||
|
"addVectorEmbedding": "Add vector embedding",
|
||||||
|
"pathEmptyError": "Path should not be empty",
|
||||||
|
"pathDuplicateError": "Path is already defined",
|
||||||
|
"dimensionRangeError": "Dimension must be greater than 0 and less than or equal 4096",
|
||||||
|
"dimensionFlatIndexError": "Maximum allowed dimension for flat index is 505",
|
||||||
|
"quantizationByteSizeRangeError": "Quantization byte size must be greater than 0 and less than or equal to 512",
|
||||||
|
"indexingSearchListSizeRangeError": "Indexing search list size must be greater than or equal to 25 and less than or equal to 500"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user