diff --git a/jest.config.js b/jest.config.js index 5a1b31550..d265f685d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -174,7 +174,11 @@ module.exports = { }, // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation - transformIgnorePatterns: ["/node_modules/(?!@fluentui/react-icons)", "/externals/"], + transformIgnorePatterns: [ + "/node_modules/(?!@fluentui/react-icons|(.*)/dist/browser)/", + "/node_modules/plotly.js-cartesian-dist-min", + "/externals/", + ], // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them // unmockedModulePathPatterns: undefined, diff --git a/package-lock.json b/package-lock.json index db0aa9b88..c05feec0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "dependencies": { "@azure/arm-cosmosdb": "9.1.0", - "@azure/cosmos": "4.0.1-beta.3", + "@azure/cosmos": "4.2.0-beta.1", "@azure/cosmos-language-service": "0.0.5", "@azure/identity": "1.5.2", "@azure/ms-rest-nodeauth": "3.1.1", @@ -228,18 +228,20 @@ } }, "node_modules/@azure/abort-controller": { - "version": "1.1.0", - "license": "MIT", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", "dependencies": { - "tslib": "^2.2.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=18.0.0" } }, "node_modules/@azure/abort-controller/node_modules/tslib": { - "version": "2.6.2", - "license": "0BSD" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, "node_modules/@azure/arm-cosmosdb": { "version": "9.1.0", @@ -251,15 +253,16 @@ } }, "node_modules/@azure/core-auth": { - "version": "1.5.0", - "license": "MIT", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", + "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-util": "^1.1.0", - "tslib": "^2.2.0" + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.11.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=18.0.0" } }, "node_modules/@azure/core-auth/node_modules/tslib": { @@ -282,36 +285,61 @@ "node": ">=18.0.0" } }, - "node_modules/@azure/core-client/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "license": "MIT", + "node_modules/@azure/core-client/node_modules/tslib": { + "version": "2.6.2", + "license": "0BSD" + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.0.tgz", + "integrity": "sha512-QSoGUp4Eq/gohEFNJaUOwTN7BCc2nHTjjbm75JT0aD7W65PWM1H/tItz0GsABn22uaKyGxiMhWQLt2r+FGU89Q==", "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.8.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, - "node_modules/@azure/core-client/node_modules/tslib": { - "version": "2.6.2", - "license": "0BSD" - }, - "node_modules/@azure/core-rest-pipeline": { - "version": "1.12.2", - "license": "MIT", + "node_modules/@azure/core-rest-pipeline/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.3.0", - "@azure/logger": "^1.0.0", - "form-data": "^4.0.0", - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "tslib": "^2.2.0" + "debug": "^4.3.4" }, "engines": { - "node": ">=16.0.0" + "node": ">= 14" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" } }, "node_modules/@azure/core-rest-pipeline/node_modules/tslib": { @@ -319,13 +347,14 @@ "license": "0BSD" }, "node_modules/@azure/core-tracing": { - "version": "1.0.1", - "license": "MIT", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", + "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", "dependencies": { - "tslib": "^2.2.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=18.0.0" } }, "node_modules/@azure/core-tracing/node_modules/tslib": { @@ -333,14 +362,15 @@ "license": "0BSD" }, "node_modules/@azure/core-util": { - "version": "1.6.1", - "license": "MIT", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", + "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "tslib": "^2.2.0" + "@azure/abort-controller": "^2.0.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.0.0" } }, "node_modules/@azure/core-util/node_modules/tslib": { @@ -348,22 +378,20 @@ "license": "0BSD" }, "node_modules/@azure/cosmos": { - "version": "4.0.1-beta.3", - "license": "MIT", + "version": "4.2.0-beta.1", + "resolved": "https://registry.npmjs.org/@azure/cosmos/-/cosmos-4.2.0-beta.1.tgz", + "integrity": "sha512-mREONehm1DxjEKXGaNU6Wmpf9Ckb9IrhKFXhDFVs45pxmoEb3y2s/Ub0owuFmqlphpcS1zgtYQn5exn+lwnJuQ==", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.3.0", - "@azure/core-rest-pipeline": "^1.2.0", - "@azure/core-tracing": "^1.0.0", - "debug": "^4.1.1", + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.7.1", + "@azure/core-rest-pipeline": "^1.15.1", + "@azure/core-tracing": "^1.1.1", + "@azure/core-util": "^1.8.1", "fast-json-stable-stringify": "^2.1.0", - "jsbi": "^3.1.3", - "node-abort-controller": "^3.0.0", + "jsbi": "^4.3.0", "priorityqueuejs": "^2.0.0", - "semaphore": "^1.0.5", - "tslib": "^2.2.0", - "universal-user-agent": "^6.0.0", - "uuid": "^8.3.0" + "semaphore": "^1.1.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" @@ -11708,6 +11736,7 @@ }, "node_modules/@tootallnate/once": { "version": "2.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">= 10" @@ -19895,6 +19924,7 @@ }, "node_modules/form-data": { "version": "4.0.0", + "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -21095,6 +21125,7 @@ }, "node_modules/http-proxy-agent": { "version": "5.0.0", + "dev": true, "license": "MIT", "dependencies": { "@tootallnate/once": "2", @@ -27067,8 +27098,9 @@ } }, "node_modules/jsbi": { - "version": "3.2.5", - "license": "Apache-2.0" + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.0.tgz", + "integrity": "sha512-SnZNcinB4RIcnEyZqFPdGPVgrg2AcnykiBy0sHVJQKHYeaLUvi3Exj+iaPpLnFVkDPZIV4U0yvgC9/R4uEAZ9g==" }, "node_modules/jsbn": { "version": "0.1.1", @@ -29737,7 +29769,9 @@ }, "node_modules/node-abort-controller": { "version": "3.1.1", - "license": "MIT" + "dev": true, + "license": "MIT", + "peer": true }, "node_modules/node-addon-api": { "version": "4.3.0", diff --git a/package.json b/package.json index d174a7674..755254b2b 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "dependencies": { "@azure/arm-cosmosdb": "9.1.0", - "@azure/cosmos": "4.0.1-beta.3", + "@azure/cosmos": "4.2.0-beta.1", "@azure/cosmos-language-service": "0.0.5", "@azure/identity": "1.5.2", "@azure/ms-rest-nodeauth": "3.1.1", @@ -247,4 +247,4 @@ "printWidth": 120, "endOfLine": "auto" } -} \ No newline at end of file +} diff --git a/src/Common/Constants.ts b/src/Common/Constants.ts index 7459df960..0c8252509 100644 --- a/src/Common/Constants.ts +++ b/src/Common/Constants.ts @@ -89,6 +89,7 @@ export class CapabilityNames { public static readonly EnableMongo: string = "EnableMongo"; public static readonly EnableServerless: string = "EnableServerless"; public static readonly EnableNoSQLVectorSearch: string = "EnableNoSQLVectorSearch"; + public static readonly EnableNoSQLFullTextSearch: string = "EnableNoSQLFullTextSearch"; } export enum CapacityMode { diff --git a/src/Common/dataAccess/createCollection.ts b/src/Common/dataAccess/createCollection.ts index b64a3add5..b5afd70a3 100644 --- a/src/Common/dataAccess/createCollection.ts +++ b/src/Common/dataAccess/createCollection.ts @@ -99,6 +99,9 @@ const createSqlContainer = async (params: DataModels.CreateCollectionParams): Pr if (params.vectorEmbeddingPolicy) { resource.vectorEmbeddingPolicy = params.vectorEmbeddingPolicy; } + if (params.fullTextPolicy) { + resource.fullTextPolicy = params.fullTextPolicy; + } const rpPayload: ARMTypes.SqlDatabaseCreateUpdateParameters = { properties: { @@ -270,6 +273,7 @@ const createCollectionWithSDK = async (params: DataModels.CreateCollectionParams uniqueKeyPolicy: params.uniqueKeyPolicy || undefined, analyticalStorageTtl: params.analyticalStorageTtl, vectorEmbeddingPolicy: params.vectorEmbeddingPolicy, + fullTextPolicy: params.fullTextPolicy, } as ContainerRequest; // TODO: remove cast when https://github.com/Azure/azure-cosmos-js/issues/423 is fixed const collectionOptions: RequestOptions = {}; const createDatabaseBody: DatabaseRequest = { id: params.databaseId }; diff --git a/src/Contracts/DataModels.ts b/src/Contracts/DataModels.ts index 3f7e3a7db..da6ddd28d 100644 --- a/src/Contracts/DataModels.ts +++ b/src/Contracts/DataModels.ts @@ -159,6 +159,7 @@ export interface Collection extends Resource { analyticalStorageTtl?: number; geospatialConfig?: GeospatialConfig; vectorEmbeddingPolicy?: VectorEmbeddingPolicy; + fullTextPolicy?: FullTextPolicy; schema?: ISchema; requestSchema?: () => void; computedProperties?: ComputedProperties; @@ -199,11 +200,19 @@ export interface IndexingPolicy { compositeIndexes?: any[]; spatialIndexes?: any[]; vectorIndexes?: VectorIndex[]; + fullTextIndexes?: FullTextIndex[]; } export interface VectorIndex { path: string; type: "flat" | "diskANN" | "quantizedFlat"; + diskANNShardKey?: string; + indexingSearchListSize?: number; + quantizationByteSize?: number; +} + +export interface FullTextIndex { + path: string; } export interface ComputedProperty { @@ -342,6 +351,7 @@ export interface CreateCollectionParams { uniqueKeyPolicy?: UniqueKeyPolicy; createMongoWildcardIndex?: boolean; vectorEmbeddingPolicy?: VectorEmbeddingPolicy; + fullTextPolicy?: FullTextPolicy; } export interface VectorEmbeddingPolicy { @@ -355,6 +365,16 @@ export interface VectorEmbedding { path: string; } +export interface FullTextPolicy { + defaultLanguage: string; + fullTextPaths: FullTextPath[]; +} + +export interface FullTextPath { + path: string; + language: string; +} + export interface ReadDatabaseOfferParams { databaseId: string; databaseResourceId?: string; diff --git a/src/Contracts/ViewModels.ts b/src/Contracts/ViewModels.ts index 09e03b2d6..b30435dc6 100644 --- a/src/Contracts/ViewModels.ts +++ b/src/Contracts/ViewModels.ts @@ -126,6 +126,8 @@ export interface Collection extends CollectionBase { analyticalStorageTtl: ko.Observable; schema?: DataModels.ISchema; requestSchema?: () => void; + vectorEmbeddingPolicy: ko.Observable; + fullTextPolicy: ko.Observable; indexingPolicy: ko.Observable; uniqueKeyPolicy: DataModels.UniqueKeyPolicy; usageSizeInKB: ko.Observable; diff --git a/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx b/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx index 80da43414..3f0fa6d2c 100644 --- a/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx +++ b/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx @@ -1,4 +1,4 @@ -import { DirectionalHint, Icon, Label, Stack, TooltipHost } from "@fluentui/react"; +import { DirectionalHint, Icon, IconButton, Label, Stack, TooltipHost } from "@fluentui/react"; import * as React from "react"; import { NormalizedEventKey } from "../../../Common/Constants"; import { accordionStackTokens } from "../Settings/SettingsRenderUtils"; @@ -9,6 +9,9 @@ export interface CollapsibleSectionProps { onExpand?: () => void; children: JSX.Element; tooltipContent?: string | JSX.Element | JSX.Element[]; + showDelete?: boolean; + onDelete?: () => void; + disabled?: boolean; } export interface CollapsibleSectionState { @@ -69,6 +72,20 @@ export class CollapsibleSectionComponent extends React.Component )} + {this.props.showDelete && ( + + { + event.stopPropagation(); + this.props.onDelete(); + }} + /> + + )} {this.state.isExpanded && this.props.children} diff --git a/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.test.tsx b/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.test.tsx new file mode 100644 index 000000000..91a2efd7f --- /dev/null +++ b/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.test.tsx @@ -0,0 +1,6 @@ +import "@testing-library/jest-dom"; + +describe("AddFullTextPolicyForm", () => { + //CTODO: add tests + it.skip("should render correctly", () => {}); +}); diff --git a/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx b/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx new file mode 100644 index 000000000..8972791ce --- /dev/null +++ b/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx @@ -0,0 +1,239 @@ +import { + DefaultButton, + Dropdown, + IDropdownOption, + IStyleFunctionOrObject, + ITextFieldStyleProps, + ITextFieldStyles, + Label, + Stack, + TextField, +} from "@fluentui/react"; +import { FullTextIndex, FullTextPath, FullTextPolicy } from "Contracts/DataModels"; +import { CollapsibleSectionComponent } from "Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent"; +import * as React from "react"; + +export interface FullTextPoliciesComponentProps { + fullTextPolicy: FullTextPolicy; + onFullTextPathChange: ( + fullTextPolicy: FullTextPolicy, + fullTextIndexes: FullTextIndex[], + validationPassed: boolean, + ) => void; + discardChanges?: boolean; + onChangesDiscarded?: () => void; +} + +export interface FullTextPolicyData { + path: string; + language: string; + pathError: string; +} + +const labelStyles = { + root: { + fontSize: 12, + }, +}; + +const textFieldStyles: IStyleFunctionOrObject = { + fieldGroup: { + height: 27, + }, + field: { + fontSize: 12, + padding: "0 8px", + }, +}; + +const dropdownStyles = { + title: { + height: 27, + lineHeight: "24px", + fontSize: 12, + }, + dropdown: { + height: 27, + lineHeight: "24px", + }, + dropdownItem: { + fontSize: 12, + }, +}; + +export const FullTextPoliciesComponent: React.FunctionComponent = ({ + fullTextPolicy, + onFullTextPathChange, + discardChanges, + onChangesDiscarded, +}): JSX.Element => { + const getFullTextPathError = (path: string, index?: number): string => { + let error = ""; + if (!path) { + error = "Full text path should not be empty"; + } + if ( + index >= 0 && + fullTextPathData?.find( + (fullTextPath: FullTextPolicyData, dataIndex: number) => dataIndex !== index && fullTextPath.path === path, + ) + ) { + error = "Full text path is already defined"; + } + return error; + }; + + const initializeData = (fullTextPolicy: FullTextPolicy): FullTextPolicyData[] => { + if (!fullTextPolicy) { + fullTextPolicy = { defaultLanguage: getFullTextLanguageOptions()[0].key as never, fullTextPaths: [] }; + } + return fullTextPolicy.fullTextPaths.map((fullTextPath: FullTextPath) => ({ + ...fullTextPath, + pathError: getFullTextPathError(fullTextPath.path), + })); + }; + + const [fullTextPathData, setFullTextPathData] = React.useState(initializeData(fullTextPolicy)); + const [defaultLanguage, setDefaultLanguage] = React.useState( + fullTextPolicy ? fullTextPolicy.defaultLanguage : (getFullTextLanguageOptions()[0].key as never), + ); + + React.useEffect(() => { + propagateData(); + }, [fullTextPathData, defaultLanguage]); + + React.useEffect(() => { + if (discardChanges) { + setFullTextPathData(initializeData(fullTextPolicy)); + setDefaultLanguage(fullTextPolicy.defaultLanguage); + onChangesDiscarded(); + } + }, [discardChanges]); + + const propagateData = () => { + const newFullTextPolicy: FullTextPolicy = { + defaultLanguage: defaultLanguage, + fullTextPaths: fullTextPathData.map((policy: FullTextPolicyData) => ({ + path: policy.path, + language: policy.language, + })), + }; + const fullTextIndexes: FullTextIndex[] = fullTextPathData.map((policy) => ({ + path: policy.path, + })); + const validationPassed = fullTextPathData.every((policy: FullTextPolicyData) => policy.pathError === ""); + onFullTextPathChange(newFullTextPolicy, fullTextIndexes, validationPassed); + }; + + const onFullTextPathValueChange = (index: number, event: React.ChangeEvent) => { + const value = event.target.value.trim(); + const fullTextPaths = [...fullTextPathData]; + if (!fullTextPaths[index]?.path && !value.startsWith("/")) { + fullTextPaths[index].path = "/" + value; + } else { + fullTextPaths[index].path = value; + } + fullTextPaths[index].pathError = getFullTextPathError(value, index); + setFullTextPathData(fullTextPaths); + }; + + const onFullTextPathPolicyChange = (index: number, option: IDropdownOption): void => { + const policies = [...fullTextPathData]; + policies[index].language = option.key as never; + setFullTextPathData(policies); + }; + + const onAdd = () => { + setFullTextPathData([ + ...fullTextPathData, + { + path: "", + language: defaultLanguage, + pathError: getFullTextPathError(""), + }, + ]); + }; + + const onDelete = (index: number) => { + const policies = fullTextPathData.filter((_uniqueKey, j) => index !== j); + setFullTextPathData(policies); + }; + + return ( + + + + , option: IDropdownOption) => + setDefaultLanguage(option.key as never) + } + > + + {fullTextPathData && + fullTextPathData.length > 0 && + fullTextPathData.map((fullTextPolicy: FullTextPolicyData, index: number) => ( + onDelete(index)} + > + + + + + ) => onFullTextPathValueChange(index, event)} + value={fullTextPolicy.path || ""} + errorMessage={fullTextPolicy.pathError} + /> + + + + , option: IDropdownOption) => + onFullTextPathPolicyChange(index, option) + } + > + + + + + ))} + + Add full text path + + + ); +}; + +export const getFullTextLanguageOptions = (): IDropdownOption[] => { + return [ + { + key: "en-US", + text: "English (US)", + }, + ]; +}; diff --git a/src/Explorer/Controls/Settings/SettingsComponent.tsx b/src/Explorer/Controls/Settings/SettingsComponent.tsx index 74f3fb4f6..57bf5b13e 100644 --- a/src/Explorer/Controls/Settings/SettingsComponent.tsx +++ b/src/Explorer/Controls/Settings/SettingsComponent.tsx @@ -4,11 +4,11 @@ import { ComputedPropertiesComponentProps, } from "Explorer/Controls/Settings/SettingsSubComponents/ComputedPropertiesComponent"; import { - ContainerVectorPolicyComponent, - ContainerVectorPolicyComponentProps, -} from "Explorer/Controls/Settings/SettingsSubComponents/ContainerVectorPolicyComponent"; + ContainerPolicyComponent, + ContainerPolicyComponentProps, +} from "Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent"; import { useDatabases } from "Explorer/useDatabases"; -import { isVectorSearchEnabled } from "Utils/CapabilityUtils"; +import { isFullTextSearchEnabled, isVectorSearchEnabled } from "Utils/CapabilityUtils"; import { isRunningOnPublicCloud } from "Utils/CloudUtils"; import * as React from "react"; import DiscardIcon from "../../../../images/discard.svg"; @@ -105,6 +105,13 @@ export interface SettingsComponentState { isSubSettingsSaveable: boolean; isSubSettingsDiscardable: boolean; + vectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy; + vectorEmbeddingPolicyBaseline: DataModels.VectorEmbeddingPolicy; + fullTextPolicy: DataModels.FullTextPolicy; + fullTextPolicyBaseline: DataModels.FullTextPolicy; + shouldDiscardContainerPolicies: boolean; + isContainerPolicyDirty: boolean; + indexingPolicyContent: DataModels.IndexingPolicy; indexingPolicyContentBaseline: DataModels.IndexingPolicy; shouldDiscardIndexingPolicy: boolean; @@ -149,6 +156,7 @@ export class SettingsComponent extends React.Component this.setState({ isScaleDiscardable: isScaleDiscardable }); + private onVectorEmbeddingPolicyChange = (newVectorEmbeddingPolicy: DataModels.VectorEmbeddingPolicy): void => + this.setState({ vectorEmbeddingPolicy: newVectorEmbeddingPolicy }); + + private onFullTextPolicyChange = (newFullTextPolicy: DataModels.FullTextPolicy): void => + this.setState({ fullTextPolicy: newFullTextPolicy }); + private onIndexingPolicyContentChange = (newIndexingPolicy: DataModels.IndexingPolicy): void => this.setState({ indexingPolicyContent: newIndexingPolicy }); + private resetShouldDiscardContainerPolicies = (): void => this.setState({ shouldDiscardContainerPolicies: false }); + private resetShouldDiscardIndexingPolicy = (): void => this.setState({ shouldDiscardIndexingPolicy: false }); private logIndexingPolicySuccessMessage = (): void => { @@ -538,6 +568,12 @@ export class SettingsComponent extends React.Component this.setState({ isSubSettingsDiscardable: isSubSettingsDiscardable }); + private onVectorEmbeddingPolicyDirtyChange = (isVectorEmbeddingPolicyDirty: boolean): void => + this.setState({ isContainerPolicyDirty: isVectorEmbeddingPolicyDirty }); + + private onFullTextPolicyDirtyChange = (isFullTextPolicyDirty: boolean): void => + this.setState({ isContainerPolicyDirty: isFullTextPolicyDirty }); + private onIndexingPolicyDirtyChange = (isIndexingPolicyDirty: boolean): void => this.setState({ isIndexingPolicyDirty: isIndexingPolicyDirty }); @@ -691,6 +727,10 @@ export class SettingsComponent extends React.Component, }); - if (this.isVectorSearchEnabled) { + if (this.isVectorSearchEnabled || this.isFullTextSearchEnabled) { tabs.push({ tab: SettingsV2TabTypes.ContainerVectorPolicyTab, - content: , + content: , }); } diff --git a/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.text.tsx b/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.text.tsx new file mode 100644 index 000000000..0ef5250c6 --- /dev/null +++ b/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.text.tsx @@ -0,0 +1,6 @@ +import "@testing-library/jest-dom"; + +describe("ContainerPolicyComponent", () => { + //CTODO: add tests + it.skip("should render correctly", () => {}); +}); diff --git a/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.tsx b/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.tsx new file mode 100644 index 000000000..f4db62ec6 --- /dev/null +++ b/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerPolicyComponent.tsx @@ -0,0 +1,163 @@ +import { DefaultButton, Pivot, PivotItem, Stack } from "@fluentui/react"; +import { FullTextPolicy, VectorEmbedding, VectorEmbeddingPolicy } from "Contracts/DataModels"; +import { + FullTextPoliciesComponent, + getFullTextLanguageOptions, +} from "Explorer/Controls/FullTextSeach/FullTextPoliciesComponent"; +import { titleAndInputStackProps } from "Explorer/Controls/Settings/SettingsRenderUtils"; +import { ContainerPolicyTabTypes, isDirty } from "Explorer/Controls/Settings/SettingsUtils"; +import { VectorEmbeddingPoliciesComponent } from "Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent"; +import React from "react"; + +export interface ContainerPolicyComponentProps { + vectorEmbeddingPolicy: VectorEmbeddingPolicy; + vectorEmbeddingPolicyBaseline: VectorEmbeddingPolicy; + onVectorEmbeddingPolicyChange: (newVectorEmbeddingPolicy: VectorEmbeddingPolicy) => void; + onVectorEmbeddingPolicyDirtyChange: (isVectorEmbeddingPolicyDirty: boolean) => void; + isVectorSearchEnabled: boolean; + fullTextPolicy: FullTextPolicy; + fullTextPolicyBaseline: FullTextPolicy; + onFullTextPolicyChange: (newFullTextPolicy: FullTextPolicy) => void; + onFullTextPolicyDirtyChange: (isFullTextPolicyDirty: boolean) => void; + isFullTextSearchEnabled: boolean; + shouldDiscardContainerPolicies: boolean; + resetShouldDiscardContainerPolicyChange: () => void; +} + +export const ContainerPolicyComponent: React.FC = ({ + vectorEmbeddingPolicy, + vectorEmbeddingPolicyBaseline, + onVectorEmbeddingPolicyChange, + onVectorEmbeddingPolicyDirtyChange, + isVectorSearchEnabled, + fullTextPolicy, + fullTextPolicyBaseline, + onFullTextPolicyChange, + onFullTextPolicyDirtyChange, + isFullTextSearchEnabled, + shouldDiscardContainerPolicies, + resetShouldDiscardContainerPolicyChange, +}) => { + const [selectedTab, setSelectedTab] = React.useState( + ContainerPolicyTabTypes.VectorPolicyTab, + ); + const [vectorEmbeddings, setVectorEmbeddings] = React.useState(); + const [vectorEmbeddingsBaseline, setVectorEmbeddingsBaseline] = React.useState(); + const [discardVectorChanges, setDiscardVectorChanges] = React.useState(false); + const [fullTextSearchPolicy, setFullTextSearchPolicy] = React.useState(); + const [fullTextSearchPolicyBaseline, setFullTextSearchPolicyBaseline] = React.useState(); + const [discardFullTextChanges, setDiscardFullTextChanges] = React.useState(false); + + React.useEffect(() => { + setVectorEmbeddings(vectorEmbeddingPolicy?.vectorEmbeddings); + setVectorEmbeddingsBaseline(vectorEmbeddingPolicyBaseline?.vectorEmbeddings); + }, [vectorEmbeddingPolicy]); + + React.useEffect(() => { + setFullTextSearchPolicy(fullTextPolicy); + setFullTextSearchPolicyBaseline(fullTextPolicyBaseline); + }, [fullTextPolicy, fullTextPolicyBaseline]); + + React.useEffect(() => { + if (shouldDiscardContainerPolicies) { + setVectorEmbeddings(vectorEmbeddingPolicyBaseline?.vectorEmbeddings); + setDiscardVectorChanges(true); + setFullTextSearchPolicy(fullTextPolicyBaseline); + setDiscardFullTextChanges(true); + resetShouldDiscardContainerPolicyChange(); + } + }); + + const checkAndSendVectorEmbeddingPoliciesToSettings = (newVectorEmbeddings: VectorEmbedding[]): void => { + if (isDirty(newVectorEmbeddings, vectorEmbeddingsBaseline)) { + onVectorEmbeddingPolicyDirtyChange(true); + onVectorEmbeddingPolicyChange({ vectorEmbeddings: newVectorEmbeddings }); + } else { + resetShouldDiscardContainerPolicyChange(); + } + }; + + const checkAndSendFullTextPolicyToSettings = (newFullTextPolicy: FullTextPolicy): void => { + if (isDirty(newFullTextPolicy, fullTextSearchPolicyBaseline)) { + onFullTextPolicyDirtyChange(true); + onFullTextPolicyChange(newFullTextPolicy); + } else { + resetShouldDiscardContainerPolicyChange(); + } + }; + + const onVectorChangesDiscarded = (): void => { + setDiscardVectorChanges(false); + }; + + const onFullTextChangesDiscarded = (): void => { + setDiscardFullTextChanges(false); + }; + + const onPivotChange = (item: PivotItem): void => { + const selectedTab = ContainerPolicyTabTypes[item.props.itemKey as keyof typeof ContainerPolicyTabTypes]; + setSelectedTab(selectedTab); + }; + + return ( +
+ + {isVectorSearchEnabled && ( + + + {vectorEmbeddings && ( + + checkAndSendVectorEmbeddingPoliciesToSettings(vectorEmbeddings) + } + discardChanges={discardVectorChanges} + onChangesDiscarded={onVectorChangesDiscarded} + /> + )} + + + )} + {isFullTextSearchEnabled && ( + + + {fullTextSearchPolicy ? ( + + checkAndSendFullTextPolicyToSettings(newFullTextPolicy) + } + discardChanges={discardFullTextChanges} + onChangesDiscarded={onFullTextChangesDiscarded} + /> + ) : ( + { + checkAndSendFullTextPolicyToSettings({ + defaultLanguage: getFullTextLanguageOptions()[0].key as never, + fullTextPaths: [], + }); + }} + > + Create new full text search policy + + )} + + + )} + +
+ ); +}; diff --git a/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerVectorPolicyComponent.tsx b/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerVectorPolicyComponent.tsx deleted file mode 100644 index ca4d63a04..000000000 --- a/src/Explorer/Controls/Settings/SettingsSubComponents/ContainerVectorPolicyComponent.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { Stack } from "@fluentui/react"; -import { VectorEmbeddingPolicy } from "Contracts/DataModels"; -import { EditorReact } from "Explorer/Controls/Editor/EditorReact"; -import { titleAndInputStackProps } from "Explorer/Controls/Settings/SettingsRenderUtils"; -import React from "react"; - -export interface ContainerVectorPolicyComponentProps { - vectorEmbeddingPolicy: VectorEmbeddingPolicy; -} - -export const ContainerVectorPolicyComponent: React.FC = ({ - vectorEmbeddingPolicy, -}) => { - return ( - - - - ); -}; diff --git a/src/Explorer/Controls/Settings/SettingsSubComponents/IndexingPolicyComponent.tsx b/src/Explorer/Controls/Settings/SettingsSubComponents/IndexingPolicyComponent.tsx index bc5de93a4..b2e7e12c2 100644 --- a/src/Explorer/Controls/Settings/SettingsSubComponents/IndexingPolicyComponent.tsx +++ b/src/Explorer/Controls/Settings/SettingsSubComponents/IndexingPolicyComponent.tsx @@ -120,11 +120,6 @@ export class IndexingPolicyComponent extends React.Component< indexTransformationProgress={this.props.indexTransformationProgress} refreshIndexTransformationProgress={this.props.refreshIndexTransformationProgress} /> - {this.props.isVectorSearchEnabled && ( - - Container vector policies and vector indexes are not modifiable after container creation - - )} {isDirty(this.props.indexingPolicyContent, this.props.indexingPolicyContentBaseline) && ( {unsavedEditorWarningMessage("indexPolicy")} )} diff --git a/src/Explorer/Controls/Settings/SettingsUtils.tsx b/src/Explorer/Controls/Settings/SettingsUtils.tsx index cff7d1f74..fa8360b9b 100644 --- a/src/Explorer/Controls/Settings/SettingsUtils.tsx +++ b/src/Explorer/Controls/Settings/SettingsUtils.tsx @@ -4,7 +4,14 @@ import * as ViewModels from "../../../Contracts/ViewModels"; import { MongoIndex } from "../../../Utils/arm/generatedClients/cosmos/types"; const zeroValue = 0; -export type isDirtyTypes = boolean | string | number | DataModels.IndexingPolicy | DataModels.ComputedProperties; +export type isDirtyTypes = + | boolean + | string + | number + | DataModels.IndexingPolicy + | DataModels.ComputedProperties + | DataModels.VectorEmbedding[] + | DataModels.FullTextPolicy; export const TtlOff = "off"; export const TtlOn = "on"; export const TtlOnNoDefault = "on-nodefault"; @@ -50,6 +57,11 @@ export enum SettingsV2TabTypes { ContainerVectorPolicyTab, } +export enum ContainerPolicyTabTypes { + VectorPolicyTab, + FullTextPolicyTab, +} + export interface IsComponentDirtyResult { isSaveable: boolean; isDiscardable: boolean; @@ -154,7 +166,7 @@ export const getTabTitle = (tab: SettingsV2TabTypes): string => { case SettingsV2TabTypes.ComputedPropertiesTab: return "Computed Properties"; case SettingsV2TabTypes.ContainerVectorPolicyTab: - return "Container Vector Policy (preview)"; + return "Container Policies"; default: throw new Error(`Unknown tab ${tab}`); } diff --git a/src/Explorer/Controls/Settings/TestUtils.tsx b/src/Explorer/Controls/Settings/TestUtils.tsx index d0c794025..c158e5cba 100644 --- a/src/Explorer/Controls/Settings/TestUtils.tsx +++ b/src/Explorer/Controls/Settings/TestUtils.tsx @@ -46,6 +46,8 @@ export const collection = { query: "query", }, ]), + vectorEmbeddingPolicy: ko.observable({} as DataModels.VectorEmbeddingPolicy), + fullTextPolicy: ko.observable({} as DataModels.FullTextPolicy), readSettings: () => { return; }, diff --git a/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap b/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap index c1c725a6f..ea6fe2864 100644 --- a/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap +++ b/src/Explorer/Controls/Settings/__snapshots__/SettingsComponent.test.tsx.snap @@ -55,6 +55,7 @@ exports[`SettingsComponent renders 1`] = ` }, "databaseId": "test", "defaultTtl": [Function], + "fullTextPolicy": [Function], "geospatialConfig": [Function], "getDatabase": [Function], "id": [Function], @@ -71,6 +72,7 @@ exports[`SettingsComponent renders 1`] = ` "readSettings": [Function], "uniqueKeyPolicy": {}, "usageSizeInKB": [Function], + "vectorEmbeddingPolicy": [Function], } } isAutoPilotSelected={false} @@ -132,6 +134,7 @@ exports[`SettingsComponent renders 1`] = ` }, "databaseId": "test", "defaultTtl": [Function], + "fullTextPolicy": [Function], "geospatialConfig": [Function], "getDatabase": [Function], "id": [Function], @@ -148,6 +151,7 @@ exports[`SettingsComponent renders 1`] = ` "readSettings": [Function], "uniqueKeyPolicy": {}, "usageSizeInKB": [Function], + "vectorEmbeddingPolicy": [Function], } } displayedTtlSeconds="5" @@ -249,6 +253,7 @@ exports[`SettingsComponent renders 1`] = ` }, "databaseId": "test", "defaultTtl": [Function], + "fullTextPolicy": [Function], "geospatialConfig": [Function], "getDatabase": [Function], "id": [Function], @@ -265,6 +270,7 @@ exports[`SettingsComponent renders 1`] = ` "readSettings": [Function], "uniqueKeyPolicy": {}, "usageSizeInKB": [Function], + "vectorEmbeddingPolicy": [Function], } } explorer={ diff --git a/src/Explorer/Panes/VectorSearchPanel/AddVectorEmbeddingPolicyForm.test.tsx b/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.test.tsx similarity index 81% rename from src/Explorer/Panes/VectorSearchPanel/AddVectorEmbeddingPolicyForm.test.tsx rename to src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.test.tsx index b7aff69ae..dc3d53cbd 100644 --- a/src/Explorer/Panes/VectorSearchPanel/AddVectorEmbeddingPolicyForm.test.tsx +++ b/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.test.tsx @@ -2,7 +2,7 @@ import "@testing-library/jest-dom"; import { RenderResult, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { VectorEmbedding, VectorIndex } from "Contracts/DataModels"; import React from "react"; -import { AddVectorEmbeddingPolicyForm } from "./AddVectorEmbeddingPolicyForm"; +import { VectorEmbeddingPoliciesComponent } from "./VectorEmbeddingPoliciesComponent"; const mockVectorEmbedding: VectorEmbedding[] = [ { path: "/vector1", dataType: "float32", distanceFunction: "euclidean", dimensions: 0 }, @@ -17,9 +17,9 @@ describe("AddVectorEmbeddingPolicyForm", () => { beforeEach(() => { component = render( - , ); @@ -36,7 +36,7 @@ describe("AddVectorEmbeddingPolicyForm", () => { }); test("calls onDelete when delete button is clicked", async () => { - const deleteButton = component.container.querySelector("#delete-vector-policy-1"); + const deleteButton = component.container.querySelector("#delete-Vector-embedding-1"); fireEvent.click(deleteButton); expect(mockOnVectorEmbeddingChange).toHaveBeenCalled(); expect(screen.queryByText("Vector embedding 1")).toBeNull(); @@ -49,21 +49,19 @@ describe("AddVectorEmbeddingPolicyForm", () => { test("validates input correctly", async () => { fireEvent.change(screen.getByPlaceholderText("/vector1"), { target: { value: "" } }); - await waitFor(() => expect(screen.getByText("Vector embedding path should not be empty")).toBeInTheDocument(), { + await waitFor(() => expect(screen.getByText("Path should not be empty")).toBeInTheDocument(), { timeout: 1500, }); await waitFor( () => - expect( - screen.getByText("Vector embedding dimension must be greater than 0 and less than or equal 4096"), - ).toBeInTheDocument(), + expect(screen.getByText("Dimension must be greater than 0 and less than or equal 4096")).toBeInTheDocument(), { timeout: 1500, }, ); fireEvent.change(component.container.querySelector("#vector-policy-dimension-1"), { target: { value: "4096" } }); fireEvent.change(screen.getByPlaceholderText("/vector1"), { target: { value: "/vector1" } }); - await waitFor(() => expect(screen.queryByText("Vector embedding path should not be empty")).toBeNull(), { + await waitFor(() => expect(screen.queryByText("Path should not be empty")).toBeNull(), { timeout: 1500, }); await waitFor( diff --git a/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx b/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx new file mode 100644 index 000000000..03d1ab1e6 --- /dev/null +++ b/src/Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent.tsx @@ -0,0 +1,470 @@ +import { + DefaultButton, + Dropdown, + IDropdownOption, + IStyleFunctionOrObject, + ITextFieldStyleProps, + ITextFieldStyles, + Label, + Stack, + TextField, +} from "@fluentui/react"; +import { VectorEmbedding, VectorIndex } from "Contracts/DataModels"; +import { CollapsibleSectionComponent } from "Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent"; +import { + getDataTypeOptions, + getDistanceFunctionOptions, + getIndexTypeOptions, +} from "Explorer/Controls/VectorSearch/VectorSearchUtils"; +import React, { FunctionComponent, useState } from "react"; + +export interface IVectorEmbeddingPoliciesComponentProps { + vectorEmbeddings: VectorEmbedding[]; + onVectorEmbeddingChange: ( + vectorEmbeddings: VectorEmbedding[], + vectorIndexingPolicies: VectorIndex[], + validationPassed: boolean, + ) => void; + vectorIndexes?: VectorIndex[]; + discardChanges?: boolean; + onChangesDiscarded?: () => void; + disabled?: boolean; +} + +export interface VectorEmbeddingPolicyData { + path: string; + dataType: VectorEmbedding["dataType"]; + distanceFunction: VectorEmbedding["distanceFunction"]; + dimensions: number; + indexType: VectorIndex["type"] | "none"; + pathError: string; + dimensionsError: string; + diskANNShardKey?: string; + diskANNShardKeyError?: string; + indexingSearchListSize?: number; + indexingSearchListSizeError?: string; + quantizationByteSize?: number; + quantizationByteSizeError?: string; +} + +type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType"; + +const labelStyles = { + root: { + fontSize: 12, + }, +}; + +const textFieldStyles: IStyleFunctionOrObject = { + fieldGroup: { + height: 27, + }, + field: { + fontSize: 12, + padding: "0 8px", + }, +}; + +const dropdownStyles = { + title: { + height: 27, + lineHeight: "24px", + fontSize: 12, + }, + dropdown: { + height: 27, + lineHeight: "24px", + }, + dropdownItem: { + fontSize: 12, + }, +}; + +export const VectorEmbeddingPoliciesComponent: FunctionComponent = ({ + vectorEmbeddings, + vectorIndexes, + onVectorEmbeddingChange, + discardChanges, + onChangesDiscarded, + disabled, +}): JSX.Element => { + const onVectorEmbeddingPathError = (path: string, index?: number): string => { + let error = ""; + if (!path) { + error = "Path should not be empty"; + } + if ( + index >= 0 && + vectorEmbeddingPolicyData?.find( + (vectorEmbedding: VectorEmbeddingPolicyData, dataIndex: number) => + dataIndex !== index && vectorEmbedding.path === path, + ) + ) { + error = "Path is already defined"; + } + return error; + }; + + const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => { + let error = ""; + if (dimension <= 0 || dimension > 4096) { + error = "Dimension must be greater than 0 and less than or equal 4096"; + } + if (indexType === "flat" && dimension > 505) { + error = "Maximum allowed dimension for flat index is 505"; + } + return error; + }; + + const onQuantizationByteSizeError = (size: number): string => { + let error = ""; + if (size < 1 || size > 512) { + error = "Quantization byte size must be greater than 0 and less than or equal to 512"; + } + return error; + }; + + const onIndexingSearchListSizeError = (size: number): string => { + let error = ""; + if (size < 25 || size > 500) { + error = "Indexing search list size must be greater than or equal to 25 and less than or equal to 500"; + } + return error; + }; + + //TODO: no restrictions yet due to this field being removed for now. + // Uncomment and replace with validation code when field is reinstated + // const onDiskANNShardKeyError = (shardKey: string): string => { + // return ""; + // }; + + const initializeData = (vectorEmbeddings: VectorEmbedding[], vectorIndexes: VectorIndex[]) => { + const mergedData: VectorEmbeddingPolicyData[] = []; + vectorEmbeddings.forEach((embedding) => { + const matchingIndex = displayIndexes ? vectorIndexes.find((index) => index.path === embedding.path) : undefined; + mergedData.push({ + ...embedding, + indexType: matchingIndex?.type || "none", + indexingSearchListSize: matchingIndex?.indexingSearchListSize || undefined, + quantizationByteSize: matchingIndex?.quantizationByteSize || undefined, + pathError: onVectorEmbeddingPathError(embedding.path), + dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"), + }); + }); + return mergedData; + }; + + const [displayIndexes] = useState(!!vectorIndexes); + const [vectorEmbeddingPolicyData, setVectorEmbeddingPolicyData] = useState( + initializeData(vectorEmbeddings, vectorIndexes), + ); + + React.useEffect(() => { + propagateData(); + }, [vectorEmbeddingPolicyData]); + + React.useEffect(() => { + if (discardChanges) { + setVectorEmbeddingPolicyData(initializeData(vectorEmbeddings, vectorIndexes)); + onChangesDiscarded(); + } + }, [discardChanges]); + + const propagateData = () => { + const vectorEmbeddings: VectorEmbedding[] = vectorEmbeddingPolicyData.map((policy: VectorEmbeddingPolicyData) => ({ + path: policy.path, + dataType: policy.dataType, + dimensions: policy.dimensions, + distanceFunction: policy.distanceFunction, + })); + const vectorIndexes: VectorIndex[] = vectorEmbeddingPolicyData + .filter((policy: VectorEmbeddingPolicyData) => policy.indexType !== "none") + .map( + (policy) => + ({ + path: policy.path, + type: policy.indexType, + indexingSearchListSize: policy.indexingSearchListSize, + quantizationByteSize: policy.quantizationByteSize, + }) as VectorIndex, + ); + const validationPassed = vectorEmbeddingPolicyData.every( + (policy: VectorEmbeddingPolicyData) => policy.pathError === "" && policy.dimensionsError === "", + ); + onVectorEmbeddingChange(vectorEmbeddings, vectorIndexes, validationPassed); + }; + + const onVectorEmbeddingPathChange = (index: number, event: React.ChangeEvent) => { + const value = event.target.value.trim(); + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + if (!vectorEmbeddings[index]?.path && !value.startsWith("/")) { + vectorEmbeddings[index].path = "/" + value; + } else { + vectorEmbeddings[index].path = value; + } + const error = onVectorEmbeddingPathError(value, index); + vectorEmbeddings[index].pathError = error; + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + const onVectorEmbeddingDimensionsChange = (index: number, event: React.ChangeEvent) => { + const value = parseInt(event.target.value.trim()) || 0; + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + const vectorEmbedding = vectorEmbeddings[index]; + vectorEmbeddings[index].dimensions = value; + const error = onVectorEmbeddingDimensionError(value, vectorEmbedding.indexType); + vectorEmbeddings[index].dimensionsError = error; + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + const onVectorEmbeddingIndexTypeChange = (index: number, option: IDropdownOption): void => { + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + const vectorEmbedding = vectorEmbeddings[index]; + vectorEmbeddings[index].indexType = option.key as never; + const error = onVectorEmbeddingDimensionError(vectorEmbedding.dimensions, vectorEmbedding.indexType); + vectorEmbeddings[index].dimensionsError = error; + if (vectorEmbedding.indexType === "diskANN") { + vectorEmbedding.indexingSearchListSize = 100; + } else { + vectorEmbedding.indexingSearchListSize = undefined; + } + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + const onQuantizationByteSizeChange = (index: number, event: React.ChangeEvent) => { + const value = parseInt(event.target.value.trim()) || 0; + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + vectorEmbeddings[index].quantizationByteSize = value; + vectorEmbeddings[index].quantizationByteSizeError = onQuantizationByteSizeError(value); + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + const onIndexingSearchListSizeChange = (index: number, event: React.ChangeEvent) => { + const value = parseInt(event.target.value.trim()) || 0; + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + vectorEmbeddings[index].indexingSearchListSize = value; + vectorEmbeddings[index].indexingSearchListSizeError = onIndexingSearchListSizeError(value); + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + // TODO: uncomment after Ignite + // DiskANNShardKey was removed for Ignite due to backend problems. Leaving this here as it will be reinstated immediately after Ignite + // const onDiskANNShardKeyChange = (index: number, event: React.ChangeEvent) => { + // const value = event.target.value.trim(); + // const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + // if (!vectorEmbeddings[index]?.diskANNShardKey && !value.startsWith("/")) { + // vectorEmbeddings[index].diskANNShardKey = "/" + value; + // } else { + // vectorEmbeddings[index].diskANNShardKey = value; + // } + // const error = onDiskANNShardKeyError(value); + // vectorEmbeddings[index].diskANNShardKeyError = error; + // setVectorEmbeddingPolicyData(vectorEmbeddings); + // } + + const onVectorEmbeddingPolicyChange = ( + index: number, + option: IDropdownOption, + property: VectorEmbeddingPolicyProperty, + ): void => { + const vectorEmbeddings = [...vectorEmbeddingPolicyData]; + vectorEmbeddings[index][property] = option.key as never; + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + const onAdd = () => { + setVectorEmbeddingPolicyData([ + ...vectorEmbeddingPolicyData, + { + path: "", + dataType: "float32", + distanceFunction: "euclidean", + dimensions: 0, + indexType: "none", + pathError: onVectorEmbeddingPathError(""), + dimensionsError: onVectorEmbeddingDimensionError(0, "none"), + }, + ]); + }; + + const onDelete = (index: number) => { + const vectorEmbeddings = vectorEmbeddingPolicyData.filter((_uniqueKey, j) => index !== j); + setVectorEmbeddingPolicyData(vectorEmbeddings); + }; + + return ( + + {vectorEmbeddingPolicyData && + vectorEmbeddingPolicyData.length > 0 && + vectorEmbeddingPolicyData.map((vectorEmbeddingPolicy: VectorEmbeddingPolicyData, index: number) => ( + onDelete(index)} + > + + + + + ) => onVectorEmbeddingPathChange(index, event)} + value={vectorEmbeddingPolicy.path || ""} + errorMessage={vectorEmbeddingPolicy.pathError} + /> + + + + , option: IDropdownOption) => + onVectorEmbeddingPolicyChange(index, option, "dataType") + } + > + + + + , option: IDropdownOption) => + onVectorEmbeddingPolicyChange(index, option, "distanceFunction") + } + > + + + + ) => + onVectorEmbeddingDimensionsChange(index, event) + } + errorMessage={vectorEmbeddingPolicy.dimensionsError} + /> + + {displayIndexes && ( + + + , option: IDropdownOption) => + onVectorEmbeddingIndexTypeChange(index, option) + } + > + + + ) => + onQuantizationByteSizeChange(index, event) + } + /> + + + + ) => + onIndexingSearchListSizeChange(index, event) + } + /> + + {/*TODO: uncomment after Ignite */} + {/* DiskANNShardKey was removed for Ignite due to backend problems. Leaving this here as it will be reinstated immediately after Ignite + + + ) => + onDiskANNShardKeyChange(index, event) + } + /> + + */} + + )} + + + + ))} + + Add vector embedding + + + ); +}; diff --git a/src/Explorer/Panes/VectorSearchPanel/VectorSearchUtils.ts b/src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts similarity index 100% rename from src/Explorer/Panes/VectorSearchPanel/VectorSearchUtils.ts rename to src/Explorer/Controls/VectorSearch/VectorSearchUtils.ts diff --git a/src/Explorer/Panes/AddCollectionPanel.tsx b/src/Explorer/Panes/AddCollectionPanel.tsx index d007f7734..8a34545d2 100644 --- a/src/Explorer/Panes/AddCollectionPanel.tsx +++ b/src/Explorer/Panes/AddCollectionPanel.tsx @@ -21,7 +21,11 @@ import { getNewDatabaseSharedThroughputDefault } from "Common/DatabaseUtility"; import { getErrorMessage, getErrorStack } from "Common/ErrorHandlingUtils"; import { configContext, Platform } from "ConfigContext"; import * as DataModels from "Contracts/DataModels"; -import { AddVectorEmbeddingPolicyForm } from "Explorer/Panes/VectorSearchPanel/AddVectorEmbeddingPolicyForm"; +import { + FullTextPoliciesComponent, + getFullTextLanguageOptions, +} from "Explorer/Controls/FullTextSeach/FullTextPoliciesComponent"; +import { VectorEmbeddingPoliciesComponent } from "Explorer/Controls/VectorSearch/VectorEmbeddingPoliciesComponent"; import { useSidePanel } from "hooks/useSidePanel"; import { useTeachingBubble } from "hooks/useTeachingBubble"; import React from "react"; @@ -30,7 +34,12 @@ import { Action } from "Shared/Telemetry/TelemetryConstants"; import * as TelemetryProcessor from "Shared/Telemetry/TelemetryProcessor"; import { userContext } from "UserContext"; import { getCollectionName } from "Utils/APITypeUtils"; -import { isCapabilityEnabled, isServerlessAccount, isVectorSearchEnabled } from "Utils/CapabilityUtils"; +import { + isCapabilityEnabled, + isFullTextSearchEnabled, + isServerlessAccount, + isVectorSearchEnabled, +} from "Utils/CapabilityUtils"; import { getUpsellMessage } from "Utils/PricingUtils"; import { CollapsibleSectionComponent } from "../Controls/CollapsiblePanel/CollapsibleSectionComponent"; import { ThroughputInput } from "../Controls/ThroughputInput/ThroughputInput"; @@ -109,6 +118,9 @@ export interface AddCollectionPanelState { vectorIndexingPolicy: DataModels.VectorIndex[]; vectorEmbeddingPolicy: DataModels.VectorEmbedding[]; vectorPolicyValidated: boolean; + fullTextPolicy: DataModels.FullTextPolicy; + fullTextIndexes: DataModels.FullTextIndex[]; + fullTextPolicyValidated: boolean; } export class AddCollectionPanel extends React.Component { @@ -147,6 +159,9 @@ export class AddCollectionPanel extends React.Component - )} + {this.shouldShowFullTextSearchParameters() && ( + + { + this.scrollToSection("collapsibleFullTextPolicySectionContent"); + }} + //TODO: uncomment when learn more text becomes available + // tooltipContent={this.getContainerFullTextPolicyTooltipContent()} + > + + + { + this.setState({ fullTextPolicy, fullTextIndexes, fullTextPolicyValidated }); + }} + /> + + + + + )} {userContext.apiType !== "Tables" && ( + // Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore + // magna aliqua.{" "} + // + // Learn more + // + // + // ); + // } + private shouldShowCollectionThroughputInput(): boolean { if (isServerlessAccount()) { return false; @@ -1274,6 +1330,10 @@ export class AddCollectionPanel extends React.Component void; -} - -export interface VectorEmbeddingPolicyData { - path: string; - dataType: VectorEmbedding["dataType"]; - distanceFunction: VectorEmbedding["distanceFunction"]; - dimensions: number; - indexType: VectorIndex["type"] | "none"; - pathError: string; - dimensionsError: string; -} - -type VectorEmbeddingPolicyProperty = "dataType" | "distanceFunction" | "indexType"; - -const textFieldStyles: IStyleFunctionOrObject = { - fieldGroup: { - height: 27, - }, - field: { - fontSize: 12, - padding: "0 8px", - }, -}; - -const dropdownStyles = { - title: { - height: 27, - lineHeight: "24px", - fontSize: 12, - }, - dropdown: { - height: 27, - lineHeight: "24px", - }, - dropdownItem: { - fontSize: 12, - }, -}; - -export const AddVectorEmbeddingPolicyForm: FunctionComponent = ({ - vectorEmbedding, - vectorIndex, - onVectorEmbeddingChange, -}): JSX.Element => { - const onVectorEmbeddingPathError = (path: string, index?: number): string => { - let error = ""; - if (!path) { - error = "Vector embedding path should not be empty"; - } - if ( - index >= 0 && - vectorEmbeddingPolicyData?.find( - (vectorEmbedding: VectorEmbeddingPolicyData, dataIndex: number) => - dataIndex !== index && vectorEmbedding.path === path, - ) - ) { - error = "Vector embedding path is already defined"; - } - return error; - }; - - const onVectorEmbeddingDimensionError = (dimension: number, indexType: VectorIndex["type"] | "none"): string => { - let error = ""; - if (dimension <= 0 || dimension > 4096) { - error = "Vector embedding dimension must be greater than 0 and less than or equal 4096"; - } - if (indexType === "flat" && dimension > 505) { - error = "Maximum allowed dimension for flat index is 505"; - } - return error; - }; - - const initializeData = (vectorEmbedding: VectorEmbedding[], vectorIndex: VectorIndex[]) => { - const mergedData: VectorEmbeddingPolicyData[] = []; - vectorEmbedding.forEach((embedding) => { - const matchingIndex = vectorIndex.find((index) => index.path === embedding.path); - mergedData.push({ - ...embedding, - indexType: matchingIndex?.type || "none", - pathError: onVectorEmbeddingPathError(embedding.path), - dimensionsError: onVectorEmbeddingDimensionError(embedding.dimensions, matchingIndex?.type || "none"), - }); - }); - return mergedData; - }; - - const [vectorEmbeddingPolicyData, setVectorEmbeddingPolicyData] = useState( - initializeData(vectorEmbedding, vectorIndex), - ); - - React.useEffect(() => { - propagateData(); - }, [vectorEmbeddingPolicyData]); - - const propagateData = () => { - const vectorEmbeddings: VectorEmbedding[] = vectorEmbeddingPolicyData.map((policy: VectorEmbeddingPolicyData) => ({ - dataType: policy.dataType, - dimensions: policy.dimensions, - distanceFunction: policy.distanceFunction, - path: policy.path, - })); - const vectorIndexingPolicies: VectorIndex[] = vectorEmbeddingPolicyData - .filter((policy: VectorEmbeddingPolicyData) => policy.indexType !== "none") - .map( - (policy) => - ({ - path: policy.path, - type: policy.indexType, - }) as VectorIndex, - ); - const validationPassed = vectorEmbeddingPolicyData.every( - (policy: VectorEmbeddingPolicyData) => policy.pathError === "" && policy.dimensionsError === "", - ); - onVectorEmbeddingChange(vectorEmbeddings, vectorIndexingPolicies, validationPassed); - }; - - const onVectorEmbeddingPathChange = (index: number, event: React.ChangeEvent) => { - const value = event.target.value.trim(); - const vectorEmbeddings = [...vectorEmbeddingPolicyData]; - if (!vectorEmbeddings[index]?.path && !value.startsWith("/")) { - vectorEmbeddings[index].path = "/" + value; - } else { - vectorEmbeddings[index].path = value; - } - const error = onVectorEmbeddingPathError(value, index); - vectorEmbeddings[index].pathError = error; - setVectorEmbeddingPolicyData(vectorEmbeddings); - }; - - const onVectorEmbeddingDimensionsChange = (index: number, event: React.ChangeEvent) => { - const value = parseInt(event.target.value.trim()) || 0; - const vectorEmbeddings = [...vectorEmbeddingPolicyData]; - const vectorEmbedding = vectorEmbeddings[index]; - vectorEmbeddings[index].dimensions = value; - const error = onVectorEmbeddingDimensionError(value, vectorEmbedding.indexType); - vectorEmbeddings[index].dimensionsError = error; - setVectorEmbeddingPolicyData(vectorEmbeddings); - }; - - const onVectorEmbeddingIndexTypeChange = (index: number, option: IDropdownOption): void => { - const vectorEmbeddings = [...vectorEmbeddingPolicyData]; - const vectorEmbedding = vectorEmbeddings[index]; - vectorEmbeddings[index].indexType = option.key as never; - const error = onVectorEmbeddingDimensionError(vectorEmbedding.dimensions, vectorEmbedding.indexType); - vectorEmbeddings[index].dimensionsError = error; - setVectorEmbeddingPolicyData(vectorEmbeddings); - }; - - const onVectorEmbeddingPolicyChange = ( - index: number, - option: IDropdownOption, - property: VectorEmbeddingPolicyProperty, - ): void => { - const vectorEmbeddings = [...vectorEmbeddingPolicyData]; - vectorEmbeddings[index][property] = option.key as never; - setVectorEmbeddingPolicyData(vectorEmbeddings); - }; - - const onAdd = () => { - setVectorEmbeddingPolicyData([ - ...vectorEmbeddingPolicyData, - { - path: "", - dataType: "float32", - distanceFunction: "euclidean", - dimensions: 0, - indexType: "none", - pathError: onVectorEmbeddingPathError(""), - dimensionsError: onVectorEmbeddingDimensionError(0, "none"), - }, - ]); - }; - - const onDelete = (index: number) => { - const vectorEmbeddings = vectorEmbeddingPolicyData.filter((_uniqueKey, j) => index !== j); - setVectorEmbeddingPolicyData(vectorEmbeddings); - }; - - return ( - - {vectorEmbeddingPolicyData.length > 0 && - vectorEmbeddingPolicyData.map((vectorEmbeddingPolicy: VectorEmbeddingPolicyData, index: number) => ( - - - - - - ) => onVectorEmbeddingPathChange(index, event)} - value={vectorEmbeddingPolicy.path || ""} - errorMessage={vectorEmbeddingPolicy.pathError} - /> - - - - , option: IDropdownOption) => - onVectorEmbeddingPolicyChange(index, option, "dataType") - } - > - - - - , option: IDropdownOption) => - onVectorEmbeddingPolicyChange(index, option, "distanceFunction") - } - > - - - - ) => - onVectorEmbeddingDimensionsChange(index, event) - } - errorMessage={vectorEmbeddingPolicy.dimensionsError} - /> - - - - , option: IDropdownOption) => - onVectorEmbeddingIndexTypeChange(index, option) - } - > - - - onDelete(index)} - /> - - - ))} - - Add vector embedding - - - ); -}; diff --git a/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2.test.tsx b/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2.test.tsx index 070ebc8e3..b0c6514ef 100644 --- a/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2.test.tsx +++ b/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2.test.tsx @@ -49,6 +49,7 @@ jest.mock("Common/dataAccess/queryDocuments", () => ({ requestCharge: 1, activityId: "activityId", indexMetrics: "indexMetrics", + correlatedActivityId: undefined, }), })), })); diff --git a/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2Mongo.test.tsx b/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2Mongo.test.tsx index 6988f448d..5628e2c7d 100644 --- a/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2Mongo.test.tsx +++ b/src/Explorer/Tabs/DocumentsTabV2/DocumentsTabV2Mongo.test.tsx @@ -67,6 +67,13 @@ jest.mock("Explorer/Controls/Dialog", () => ({ }, })); +// Added as recent change to @azure/core-util would cause randomUUID() to throw an error during jest tests. +// TODO: when not using beta version of @azure/cosmos sdk try removing this +jest.mock("@azure/core-util", () => ({ + ...jest.requireActual("@azure/core-util"), + randomUUID: jest.fn(), +})); + async function waitForComponentToPaint

(wrapper: ReactWrapper

| ShallowWrapper

, amount = 0) { let newWrapper; await act(async () => { diff --git a/src/Explorer/Tree/Collection.ts b/src/Explorer/Tree/Collection.ts index 26246f1b1..8015f7643 100644 --- a/src/Explorer/Tree/Collection.ts +++ b/src/Explorer/Tree/Collection.ts @@ -52,6 +52,8 @@ export default class Collection implements ViewModels.Collection { public partitionKeyProperties: string[]; public id: ko.Observable; public defaultTtl: ko.Observable; + public vectorEmbeddingPolicy: ko.Observable; + public fullTextPolicy: ko.Observable; public indexingPolicy: ko.Observable; public uniqueKeyPolicy: DataModels.UniqueKeyPolicy; public usageSizeInKB: ko.Observable; @@ -110,6 +112,8 @@ export default class Collection implements ViewModels.Collection { this.id = ko.observable(data.id); this.defaultTtl = ko.observable(data.defaultTtl); + this.vectorEmbeddingPolicy = ko.observable(data.vectorEmbeddingPolicy); + this.fullTextPolicy = ko.observable(data.fullTextPolicy); this.indexingPolicy = ko.observable(data.indexingPolicy); this.usageSizeInKB = ko.observable(); this.offer = ko.observable(); diff --git a/src/Utils/CapabilityUtils.ts b/src/Utils/CapabilityUtils.ts index 8b6976666..fdfac1bd4 100644 --- a/src/Utils/CapabilityUtils.ts +++ b/src/Utils/CapabilityUtils.ts @@ -20,3 +20,7 @@ export const isServerlessAccount = (): boolean => { export const isVectorSearchEnabled = (): boolean => { return userContext.apiType === "SQL" && isCapabilityEnabled(Constants.CapabilityNames.EnableNoSQLVectorSearch); }; + +export const isFullTextSearchEnabled = (): boolean => { + return userContext.apiType === "SQL" && isCapabilityEnabled(Constants.CapabilityNames.EnableNoSQLFullTextSearch); +}; diff --git a/src/Utils/arm/generatedClients/cosmos/types.ts b/src/Utils/arm/generatedClients/cosmos/types.ts index 5272f215b..1871884e0 100644 --- a/src/Utils/arm/generatedClients/cosmos/types.ts +++ b/src/Utils/arm/generatedClients/cosmos/types.ts @@ -1237,6 +1237,7 @@ export interface SqlContainerResource { id: string; vectorEmbeddingPolicy?: VectorEmbeddingPolicy; + fullTextPolicy?: FullTextPolicy; /* The configuration of the indexing policy. By default, the indexing is automatic for all document paths within the container */ indexingPolicy?: IndexingPolicy; @@ -1281,6 +1282,28 @@ export interface VectorEmbedding { distanceFunction?: string; } +export interface FullTextPolicy { + /** + * The default language for the full text . + */ + defaultLanguage: string; + /** + * The paths to be indexed for full text search. + */ + fullTextPaths: FullTextPath[]; +} + +export interface FullTextPath { + /** + * The path to be indexed for full text search. + */ + path: string; + /** + * The language for the full text path. + */ + language: string; +} + /* Cosmos DB indexing policy */ export interface IndexingPolicy { /* Indicates if the indexing policy is automatic */ @@ -1301,6 +1324,8 @@ export interface IndexingPolicy { spatialIndexes?: SpatialSpec[]; vectorIndexes?: VectorIndex[]; + + fullTextIndexes?: FullTextIndex[]; } export interface VectorIndex { @@ -1308,6 +1333,11 @@ export interface VectorIndex { type?: string; } +export interface FullTextIndex { + /** The path in the JSON document to index. */ + path: string; +} + /* undocumented */ export interface ExcludedPath { /* The path for which the indexing behavior applies to. Index paths typically start with root and end with wildcard (/path/*) */