From 226a7a27ee4f6311ee90456141ada8789540dc27 Mon Sep 17 00:00:00 2001 From: sunghyunkang1111 <114709653+sunghyunkang1111@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:31:20 -0500 Subject: [PATCH] Add configurable full-text stopword policies (#2606) --- .../dataAccess/createCollection.test.ts | 169 +++++ src/Contracts/DataModels.ts | 22 +- .../CollapsibleSectionComponent.tsx | 2 + .../FullTextPoliciesComponent.test.tsx | 540 +++++++++++++- .../FullTextPoliciesComponent.tsx | 666 ++++++++++++------ .../FullTextSeach/FullTextPolicyUtils.test.ts | 324 +++++++++ .../FullTextSeach/FullTextPolicyUtils.ts | 121 ++++ .../FullTextSeach/StopwordSettings.test.tsx | 241 +++++++ .../FullTextSeach/StopwordSettings.tsx | 210 ++++++ .../Settings/SettingsComponent.test.tsx | 212 +++++- .../Controls/Settings/SettingsComponent.tsx | 42 +- .../ContainerPolicyComponent.tsx | 50 +- .../SettingsComponent.test.tsx.snap | 1 + .../AddCollectionPanel.test.tsx | 72 ++ .../AddCollectionPanel/AddCollectionPanel.tsx | 19 +- .../AddCollectionPanel.test.tsx.snap | 2 +- src/Localization/en/Resources.json | 59 ++ test/README.md | 104 +++ test/resources/account.bicep | 2 +- test/sql/container.spec.ts | 220 +++++- tsconfig.strict.json | 6 +- 21 files changed, 2827 insertions(+), 257 deletions(-) create mode 100644 src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.test.ts create mode 100644 src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.ts create mode 100644 src/Explorer/Controls/FullTextSeach/StopwordSettings.test.tsx create mode 100644 src/Explorer/Controls/FullTextSeach/StopwordSettings.tsx diff --git a/src/Common/dataAccess/createCollection.test.ts b/src/Common/dataAccess/createCollection.test.ts index 2f6bf63e4..2b07fd7b1 100644 --- a/src/Common/dataAccess/createCollection.test.ts +++ b/src/Common/dataAccess/createCollection.test.ts @@ -9,6 +9,14 @@ import { updateUserContext } from "../../UserContext"; import { armRequest } from "../../Utils/arm/request"; import { client } from "../CosmosClient"; import { constructRpOptions, createCollection } from "./createCollection"; +import { FullTextPolicy, Collection } from "Contracts/DataModels"; +import { handleError } from "Common/ErrorHandlingUtils"; +import { userContext } from "UserContext"; +import { readCollections } from "./readCollections"; +import { updateCollection } from "./updateCollection"; +import { fullTextLanguages } from "Explorer/Controls/FullTextSeach/FullTextPolicyUtils"; + +jest.mock("Common/ErrorHandlingUtils", () => ({ handleError: jest.fn() })); describe("createCollection", () => { const createCollectionParams: CreateCollectionParams = { @@ -37,6 +45,167 @@ describe("createCollection", () => { }); }); + describe("full-text policy transport", () => { + const policy: FullTextPolicy = { + defaultLanguage: "en-US", + package: "standard", + defaultSpec: { + language: "en-US", + stopWordListKind: "basic", + addStopWords: ["Cosmos", "cosmos"], + removeStopWords: ["the"], + tokenizer: "word", + filters: ["lowercase", "stop"], + futureSetting: { preserved: true }, + }, + fullTextPaths: [{ path: "/text" }, { path: "/other", language: "fr-FR", stopWordListKind: "none" }], + futurePolicySetting: "preserved", + }; + const resource: Partial = { + id: "container", + fullTextPolicy: policy, + indexingPolicy: { + automatic: true, + indexingMode: "consistent", + includedPaths: [], + excludedPaths: [], + fullTextIndexes: [{ path: "/text" }], + }, + }; + const sdkCreate = jest.fn(); + const sdkReplace = jest.fn(); + const sdkRead = jest.fn(); + + beforeEach(() => { + jest.resetAllMocks(); + updateUserContext({ + apiType: "SQL", + databaseAccount: { name: "account" } as DatabaseAccount, + subscriptionId: "subscription", + resourceGroup: "group", + features: { ...userContext.features, enableSDKoperations: false }, + }); + useDatabases.setState({ validateCollectionId: jest.fn().mockResolvedValue(true) }); + sdkCreate.mockResolvedValue({ resource }); + sdkReplace.mockResolvedValue({ resource }); + sdkRead.mockResolvedValue({ resources: [resource] }); + const database = { + containers: { create: sdkCreate, readAll: () => ({ fetchAll: sdkRead }) }, + container: () => ({ replace: sdkReplace }), + }; + (client as jest.Mock).mockReturnValue({ + databases: { createIfNotExists: jest.fn().mockResolvedValue({ database }) }, + database: () => database, + }); + jest.mocked(armRequest).mockResolvedValue({ properties: { resource } }); + }); + + it.each([AuthType.AAD, AuthType.MasterKey])( + "preserves policy creation and dedicated indexes through %s", + async (authType) => { + updateUserContext({ authType }); + const result = await createCollection({ + createNewDatabase: false, + databaseId: "database", + collectionId: "container", + databaseLevelThroughput: false, + offerThroughput: 400, + fullTextPolicy: policy, + indexingPolicy: resource.indexingPolicy, + }); + expect(result.fullTextPolicy).toEqual(policy); + const payload = + authType === AuthType.AAD + ? (armRequest as jest.Mock).mock.lastCall[0].body.properties.resource + : sdkCreate.mock.lastCall[0]; + expect(payload).toMatchObject({ fullTextPolicy: policy, indexingPolicy: resource.indexingPolicy }); + const throughput = + authType === AuthType.AAD + ? (armRequest as jest.Mock).mock.lastCall[0].body.properties.options.throughput + : sdkCreate.mock.lastCall[0].throughput; + expect(throughput).toBe(400); + }, + ); + + it.each([AuthType.AAD, AuthType.MasterKey])( + "preserves analysis through read and unrelated update with %s", + async (authType) => { + updateUserContext({ authType }); + jest.mocked(armRequest).mockResolvedValueOnce({ value: [{ properties: { resource } }] }); + expect((await readCollections("database"))[0].fullTextPolicy).toEqual(policy); + jest.mocked(armRequest).mockResolvedValue({ properties: { resource } }); + await updateCollection("database", "container", { ...resource, defaultTtl: 123 }); + const payload = + authType === AuthType.AAD + ? (armRequest as jest.Mock).mock.lastCall[0].body.properties.resource + : sdkReplace.mock.lastCall[0]; + expect(payload).toEqual({ ...resource, defaultTtl: 123 }); + }, + ); + + it.each([AuthType.AAD, AuthType.MasterKey])( + "surfaces a rejected indexed-policy update with %s", + async (authType) => { + updateUserContext({ authType }); + const failure = new Error("Cannot change analysis for an indexed path"); + jest.mocked(armRequest).mockResolvedValueOnce({ properties: { resource } }).mockRejectedValueOnce(failure); + sdkReplace.mockRejectedValueOnce(failure); + await expect(updateCollection("database", "container", resource)).rejects.toBe(failure); + expect(handleError).toHaveBeenCalledWith(failure, "UpdateCollection", expect.any(String)); + expect(resource.fullTextPolicy).toEqual(policy); + }, + ); + + it.each( + fullTextLanguages.flatMap(({ value }) => + [AuthType.AAD, AuthType.MasterKey].map((authType) => ({ language: value, authType })), + ), + )("preserves 20-word $language policies through $authType create/read/update", async ({ language, authType }) => { + const words = Array.from({ length: 20 }, (_, index) => ["catalog", "Catalog", "caf\u00e9", "catalog"][index % 4]); + const localized = { + ...policy, + defaultLanguage: language, + defaultSpec: { ...policy.defaultSpec, language, addStopWords: words, removeStopWords: words }, + }; + const current = { ...resource, fullTextPolicy: localized }; + updateUserContext({ authType }); + sdkCreate.mockResolvedValue({ resource: current }); + sdkRead.mockResolvedValue({ resources: [current] }); + sdkReplace.mockResolvedValue({ resource: current }); + jest.mocked(armRequest).mockResolvedValue({ properties: { resource: current } }); + const created = await createCollection({ + createNewDatabase: false, + databaseId: "database", + collectionId: "container", + databaseLevelThroughput: false, + offerThroughput: 400, + fullTextPolicy: localized, + indexingPolicy: current.indexingPolicy, + }); + expect(created.fullTextPolicy).toEqual(localized); + const createPayload = + authType === AuthType.AAD ? jest.mocked(armRequest).mock.lastCall[0].body : sdkCreate.mock.lastCall[0]; + expect(createPayload).toMatchObject( + authType === AuthType.AAD + ? { properties: { resource: { fullTextPolicy: localized } } } + : { fullTextPolicy: localized }, + ); + if (authType === AuthType.AAD) { + jest.mocked(armRequest).mockResolvedValueOnce({ value: [{ properties: { resource: current } }] }); + } + expect((await readCollections("database"))[0].fullTextPolicy).toEqual(localized); + jest.mocked(armRequest).mockResolvedValue({ properties: { resource: current } }); + await updateCollection("database", "container", { ...current, defaultTtl: 123 }); + const updatePayload = + authType === AuthType.AAD ? jest.mocked(armRequest).mock.lastCall[0].body : sdkReplace.mock.lastCall[0]; + expect(updatePayload).toMatchObject( + authType === AuthType.AAD + ? { properties: { resource: { ...current, defaultTtl: 123 } } } + : { ...current, defaultTtl: 123 }, + ); + }); + }); + it("should call ARM if logged in with AAD", async () => { updateUserContext({ authType: AuthType.AAD, diff --git a/src/Contracts/DataModels.ts b/src/Contracts/DataModels.ts index ad8cbdf38..289aec701 100644 --- a/src/Contracts/DataModels.ts +++ b/src/Contracts/DataModels.ts @@ -465,14 +465,26 @@ export interface VectorEmbedding { path: string; } -export interface FullTextPolicy { - defaultLanguage: string; - fullTextPaths: FullTextPath[]; +export interface FullTextAnalysisSpec { + language?: string; + stopWordListKind?: string; + addStopWords?: string[]; + removeStopWords?: string[]; + tokenizer?: string; + filters?: string[]; + [key: string]: unknown; } -export interface FullTextPath { +export interface FullTextPolicy { + defaultLanguage?: string; + package?: string; + defaultSpec?: FullTextAnalysisSpec; + fullTextPaths: FullTextPath[]; + [key: string]: unknown; +} + +export interface FullTextPath extends FullTextAnalysisSpec { path: string; - language: string; } export interface ReadDatabaseOfferParams { diff --git a/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx b/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx index 3dcea02ac..57dac028a 100644 --- a/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx +++ b/src/Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent.tsx @@ -13,6 +13,7 @@ export interface CollapsibleSectionProps { onDelete?: () => void; disabled?: boolean; disableDelete?: boolean; + deleteLabel?: string; } export interface CollapsibleSectionState { @@ -76,6 +77,7 @@ export class CollapsibleSectionComponent extends React.Component { - //CTODO: add tests - it.skip("should render correctly", () => {}); +describe("FullTextPoliciesComponent", () => { + const account: DatabaseAccount = { + id: "test", + name: "test", + type: "Microsoft.DocumentDB/databaseAccounts", + location: "westus", + kind: "GlobalDocumentDB", + tags: {}, + properties: { + capabilities: [{ name: "EnableNoSQLFullTextSearchPreviewFeatures", description: "" }], + }, + }; + const policy: FullTextPolicy = { + defaultLanguage: "en-US", + package: "standard", + defaultSpec: { + language: "en-US", + stopWordListKind: "basic", + addStopWords: ["cosmos"], + removeStopWords: ["the"], + tokenizer: "word", + filters: ["lowercase", "stop"], + futureField: { preserved: true }, + }, + fullTextPaths: [{ path: "/text" }, { path: "/other", language: "fr-FR", stopWordListKind: "none" }], + futurePolicyField: "preserved", + }; + + beforeEach(() => { + updateUserContext({ apiType: "SQL", databaseAccount: account }); + }); + + describe("ContainerPolicyComponent full-text editing", () => { + const baseline: FullTextPolicy = { + package: "standard", + defaultSpec: { + language: "en-US", + stopWordListKind: "basic", + filters: ["lowercase", "stop"], + addStopWords: ["cosmos"], + future: { preserve: true }, + }, + fullTextPaths: [{ path: "/text" }], + }; + const account: DatabaseAccount = { + id: "test", + name: "test", + type: "Microsoft.DocumentDB/databaseAccounts", + location: "westus", + kind: "GlobalDocumentDB", + properties: { capabilities: [{ name: "EnableNoSQLFullTextSearchPreviewFeatures", description: "" }] }, + }; + const props = (): ContainerPolicyComponentProps => ({ + vectorEmbeddingPolicy: { vectorEmbeddings: [] }, + vectorEmbeddingPolicyBaseline: { vectorEmbeddings: [] }, + onVectorEmbeddingPolicyChange: jest.fn(), + onVectorEmbeddingPolicyDirtyChange: jest.fn(), + onVectorEmbeddingPolicyValidationChange: jest.fn(), + vectorIndexes: [], + vectorIndexesBaseline: [], + onVectorIndexesChange: jest.fn(), + isVectorSearchEnabled: false, + fullTextPolicy: baseline, + fullTextPolicyBaseline: baseline, + onFullTextPolicyChange: jest.fn(), + onFullTextPolicyDirtyChange: jest.fn(), + onFullTextPolicyValidationChange: jest.fn(), + fullTextIndexesBaseline: [], + isFullTextSearchEnabled: true, + shouldDiscardContainerPolicies: false, + resetShouldDiscardContainerPolicyChange: jest.fn(), + }); + + beforeEach(() => updateUserContext({ apiType: "SQL", databaseAccount: account })); + + it("opens the full-text tab when vector search is unavailable and propagates validation and reversions", () => { + const callbacks = props(); + const Harness = () => { + const [policy, setPolicy] = React.useState(baseline); + return ( + { + callbacks.onFullTextPolicyChange(next); + setPolicy(next); + }} + /> + ); + }; + render(); + const words = within(screen.getByRole("group", { name: "Default stopwords" })).getByRole("textbox", { + name: "Additional stopwords", + }); + expect(callbacks.onFullTextPolicyDirtyChange).toHaveBeenLastCalledWith(false); + fireEvent.change(words, { target: { value: "two words" } }); + expect(callbacks.onFullTextPolicyDirtyChange).toHaveBeenLastCalledWith(true); + expect(callbacks.onFullTextPolicyValidationChange).toHaveBeenLastCalledWith(false); + fireEvent.change(words, { target: { value: "cosmos" } }); + expect(callbacks.onFullTextPolicyChange).toHaveBeenLastCalledWith(baseline); + expect(callbacks.onFullTextPolicyDirtyChange).toHaveBeenLastCalledWith(false); + expect(callbacks.onFullTextPolicyValidationChange).toHaveBeenLastCalledWith(true); + }); + + it("discards drafts without losing hidden fields or relocking against draft indexes", () => { + const callbacks = props(); + const { rerender } = render(); + const words = within(screen.getByRole("group", { name: "Default stopwords" })).getByRole("textbox", { + name: "Additional stopwords", + }); + fireEvent.change(words, { target: { value: "draft" } }); + rerender(); + expect(words).toHaveValue("cosmos"); + expect(callbacks.onFullTextPolicyChange).toHaveBeenLastCalledWith(baseline); + expect(callbacks.resetShouldDiscardContainerPolicyChange).toHaveBeenCalledTimes(1); + rerender(); + expect(words).toBeDisabled(); + expect(screen.getByRole("textbox", { name: "Path" })).toBeDisabled(); + }); + + it("propagates vector removal instead of leaving a stale draft to save with full-text edits", () => { + const vector: VectorEmbedding = { + path: "/vector", + dataType: "float32", + dimensions: 8, + distanceFunction: "cosine", + }; + const callbacks = props(); + const Harness = () => { + const [vectorPolicy, setVectorPolicy] = React.useState({ vectorEmbeddings: [vector] }); + return ( + { + callbacks.onVectorEmbeddingPolicyChange(next); + setVectorPolicy(next); + }} + /> + ); + }; + const { container } = render(); + const deleteButton = container.querySelector('[id^="delete-Vector"]'); + expect(deleteButton).not.toBeNull(); + fireEvent.click(deleteButton!); + expect(callbacks.onVectorEmbeddingPolicyChange).toHaveBeenLastCalledWith({ vectorEmbeddings: [] }); + }); + }); + + it("preserves the complete policy on mount instead of reconstructing legacy fields", () => { + const changed = jest.fn(); + render( + , + ); + expect(changed).toHaveBeenLastCalledWith(policy, [], true); + expect(screen.getByRole("group", { name: "Default stopwords" })).toBeVisible(); + }); + + it("keeps capability gating independent of the English-only language restriction", () => { + render( + , + ); + expect(screen.getByRole("group", { name: "Default stopwords" })).toBeVisible(); + expect(getFullTextLanguageOptions(true).map((option) => option.key)).toEqual(["en-US"]); + expect(getFullTextLanguageOptions().map((option) => option.key)).toHaveLength(7); + }); + + const apiTypes: ApiType[] = ["SQL", "Mongo", "Gremlin", "Tables", "Cassandra", "Postgres", "VCoreMongo"]; + it.each(apiTypes)("gates stopwords and languages correctly for %s and target-account overrides", (apiType) => { + for (const preview of [false, true]) { + const capabilities = preview + ? account.properties.capabilities + : [{ name: "EnableNoSQLFullTextSearch", description: "" }]; + updateUserContext({ apiType, databaseAccount: { ...account, properties: { capabilities } } }); + updateUserContext({ apiType }); + expect(isFullTextSearchPreviewFeaturesEnabled()).toBe(apiType === "SQL" && preview); + expect(getFullTextLanguageOptions()).toHaveLength(apiType === "SQL" && preview ? 7 : 1); + expect(getFullTextLanguageOptions(true)).toHaveLength(1); + for (const targetPreview of [false, true]) { + const target = { + subscriptionId: "subscription", + resourceGroup: "group", + accountName: "target", + capabilities: targetPreview ? account.properties.capabilities : [], + }; + expect(isFullTextSearchPreviewFeaturesEnabled(target)).toBe(apiType === "SQL" && targetPreview); + expect(getFullTextLanguageOptions(false, target)).toHaveLength(apiType === "SQL" && targetPreview ? 7 : 1); + } + } + }); + + it("preserves a custom policy when the preview capability is unavailable", () => { + updateUserContext({ databaseAccount: { ...account, properties: { capabilities: [] } } }); + const changed = jest.fn(); + render( + , + ); + expect(screen.queryByRole("group", { name: "Default stopwords" })).not.toBeInTheDocument(); + expect(screen.getByText(/customization is not enabled for this connection/)).toBeVisible(); + expect(changed).toHaveBeenLastCalledWith(policy, [], true); + }); + + it("opts into standard customization explicitly and restores a legacy-shaped new policy when unchecked", () => { + const legacy: FullTextPolicy = { defaultLanguage: "en-US", fullTextPaths: [{ path: "/text", language: "en-US" }] }; + const changed = jest.fn(); + render( + , + ); + expect(changed).toHaveBeenLastCalledWith(legacy, [{ path: "/text" }], true); + const toggle = screen.getByRole("checkbox", { name: "Customize stopwords with standard analysis" }); + fireEvent.click(toggle); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].package).toBe("standard"); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].fullTextPaths).toEqual([{ path: "/text" }]); + fireEvent.click(toggle); + expect(changed).toHaveBeenLastCalledWith(legacy, [{ path: "/text" }], true); + expect(legacy).toEqual({ defaultLanguage: "en-US", fullTextPaths: [{ path: "/text", language: "en-US" }] }); + }); + + it("inherits the current and future default filters after resetting a path override", () => { + const changed = jest.fn(); + const original: FullTextPolicy = { + ...policy, + fullTextPaths: [{ path: "/text", futurePathSetting: { preserved: true } }], + }; + render( + , + ); + const inherit = screen.getByRole("checkbox", { name: "Inherit the container's language and stopwords" }); + fireEvent.click(inherit); + const pathSettings = within(screen.getByRole("group", { name: "Stopwords for /text" })); + expect(pathSettings.getByRole("checkbox", { name: "Enable stopword filtering" })).toBeChecked(); + fireEvent.click(pathSettings.getByRole("checkbox", { name: "Enable stopword filtering" })); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].fullTextPaths[0].filters).toEqual(["lowercase"]); + fireEvent.click(inherit); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].fullTextPaths).toEqual(original.fullTextPaths); + expect(screen.queryByRole("group", { name: "Stopwords for /text" })).not.toBeInTheDocument(); + const defaults = within(screen.getByRole("group", { name: "Default stopwords" })); + fireEvent.click(defaults.getByRole("checkbox", { name: "Enable stopword filtering" })); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].fullTextPaths).toEqual(original.fullTextPaths); + fireEvent.click(inherit); + expect( + within(screen.getByRole("group", { name: "Stopwords for /text" })).getByRole("checkbox", { + name: "Enable stopword filtering", + }), + ).not.toBeChecked(); + expect(original.defaultSpec?.filters).toEqual(["lowercase", "stop"]); + }); + + it("reports analysis-only path overrides instead of hiding them as inherited", () => { + const changed = jest.fn(); + render( + , + ); + const inherit = screen.getByRole("checkbox", { name: "Inherit the container's language and stopwords" }); + expect(inherit).not.toBeChecked(); + expect(screen.getByRole("group", { name: "Stopwords for /text" })).toBeVisible(); + fireEvent.click(inherit); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].fullTextPaths).toEqual([{ path: "/text" }]); + }); + + it.each(["capability", "compatibility", "indexed"])( + "preserves uneditable package/preset combinations without blocking saves: %s", + (reason) => { + if (reason === "capability") { + updateUserContext({ databaseAccount: { ...account, properties: { capabilities: [] } } }); + } + const persisted: FullTextPolicy = { + ...policy, + defaultSpec: { ...policy.defaultSpec, stopWordListKind: "extended" }, + fullTextPaths: [{ path: "/text" }], + }; + const changed = jest.fn(); + const indexes = reason === "indexed" ? [{ path: "/text" }] : []; + render( + , + ); + expect(changed).toHaveBeenLastCalledWith(persisted, indexes, true); + expect(screen.getByRole("combobox", { name: /^Default language/ })).toHaveAttribute("aria-disabled", "true"); + expect(persisted.defaultSpec?.stopWordListKind).toBe("extended"); + }, + ); + + it("still blocks an editable invalid preset until the user corrects it", () => { + const changed = jest.fn(); + render( + , + ); + expect(changed.mock.calls[changed.mock.calls.length - 1][2]).toBe(false); + fireEvent.click( + within(screen.getByRole("group", { name: "Default stopwords" })).getByRole("combobox", { + name: "Stopword list", + }), + ); + fireEvent.click(screen.getByRole("option", { name: "Basic", exact: true })); + expect(changed.mock.calls[changed.mock.calls.length - 1][2]).toBe(true); + }); + + it.each([{ tokenizer: "future" }, { filters: ["future"] }])( + "preserves opaque path analysis without allowing an inheritance reset: %j", + (analysis) => { + const persisted: FullTextPolicy = { + ...policy, + fullTextPaths: [{ path: "/text", ...analysis, futureSetting: { preserved: true } }], + }; + const changed = jest.fn(); + render( + , + ); + expect(screen.getByRole("checkbox", { name: "Inherit the container's language and stopwords" })).toBeDisabled(); + expect(changed).toHaveBeenLastCalledWith(persisted, [], true); + }, + ); + + it("propagates invalid custom words and recovers without mutating the baseline", () => { + const changed = jest.fn(); + render( + , + ); + const input = within(screen.getByRole("group", { name: "Default stopwords" })).getByRole("textbox", { + name: "Additional stopwords", + }); + fireEvent.change(input, { target: { value: "two words" } }); + expect(changed.mock.calls[changed.mock.calls.length - 1][2]).toBe(false); + expect(screen.getByRole("alert")).toHaveTextContent("cannot contain"); + fireEvent.change(input, { target: { value: "Cosmos\ncosmos" } }); + const updated = changed.mock.calls[changed.mock.calls.length - 1][0]; + expect(updated.defaultSpec.addStopWords).toEqual(["Cosmos", "cosmos"]); + expect(updated.defaultSpec.futureField).toEqual({ preserved: true }); + expect(changed.mock.calls[changed.mock.calls.length - 1][2]).toBe(true); + expect(policy.defaultSpec?.addStopWords).toEqual(["cosmos"]); + }); + + it("locks only persisted indexed paths and global defaults, not unrelated paths", () => { + render( + , + ); + expect( + within(screen.getByRole("group", { name: "Default stopwords" })).getByRole("textbox", { + name: "Additional stopwords", + }), + ).toBeDisabled(); + const paths = screen.getAllByRole("textbox", { name: "Path" }); + expect(paths[0]).toBeDisabled(); + expect(paths[1]).toBeEnabled(); + expect( + within(screen.getByRole("group", { name: "Stopwords for /other" })).getByRole("textbox", { + name: "Additional stopwords", + }), + ).toBeEnabled(); + }); + + it("discards draft changes and restores all hidden fields from the supplied baseline", () => { + const changed = jest.fn(); + const discarded = jest.fn(); + const props = { + fullTextPolicy: policy, + onFullTextPathChange: changed, + onChangesDiscarded: discarded, + allowStopwordCustomization: true, + isEditing: true, + }; + const { rerender } = render(); + const input = within(screen.getByRole("group", { name: "Default stopwords" })).getByRole("textbox", { + name: "Additional stopwords", + }); + fireEvent.change(input, { target: { value: "changed" } }); + rerender(); + expect(input).toHaveValue("cosmos"); + expect(changed).toHaveBeenLastCalledWith(policy, [], true); + expect(discarded).toHaveBeenCalledTimes(1); + }); + + it("keeps unknown presets read-only and lossless", () => { + const future = { ...policy, defaultSpec: { ...policy.defaultSpec, stopWordListKind: "future" } }; + const changed = jest.fn(); + render( + , + ); + expect(screen.getByText(/settings this editor cannot safely change/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Add full text path" })).toBeDisabled(); + expect(changed).toHaveBeenLastCalledWith(future, [], true); + }); + + it("requires explicit activation when an existing standard policy has no stop filter", () => { + const withoutFilter: FullTextPolicy = { ...policy, defaultSpec: { ...policy.defaultSpec, filters: [] } }; + const changed = jest.fn(); + render( + , + ); + const defaults = within(screen.getByRole("group", { name: "Default stopwords" })); + expect(defaults.getByRole("textbox", { name: "Additional stopwords" })).toBeDisabled(); + fireEvent.click(defaults.getByRole("checkbox", { name: "Enable stopword filtering" })); + expect(defaults.getByRole("textbox", { name: "Additional stopwords" })).toBeEnabled(); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].defaultSpec.filters).toEqual(["stop"]); + expect(withoutFilter.defaultSpec?.filters).toEqual([]); + }); + + it("does not inherit filters when a path explicitly chooses a tokenizer and writes a tokenizer with new filters", () => { + const changed = jest.fn(); + const explicitTokenizer = { + ...policy, + fullTextPaths: [{ path: "/text", language: "en-US", tokenizer: "word", stopWordListKind: "basic" }], + }; + render( + , + ); + const pathSettings = within(screen.getByRole("group", { name: "Stopwords for /text" })); + expect(pathSettings.getByRole("checkbox", { name: "Enable stopword filtering" })).not.toBeChecked(); + fireEvent.click(pathSettings.getByRole("checkbox", { name: "Enable stopword filtering" })); + expect(changed.mock.calls[changed.mock.calls.length - 1][0].fullTextPaths[0]).toEqual({ + ...explicitTokenizer.fullTextPaths[0], + filters: ["stop"], + }); + }); + + it("preserves customized legacy policies read-only without the capability", () => { + updateUserContext({ databaseAccount: { ...account, properties: { capabilities: [] } } }); + const legacy: FullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [{ path: "/text", language: "en-US", stopWordListKind: "extended", addStopWords: ["cosmos"] }], + }; + const changed = jest.fn(); + render( + , + ); + expect(screen.getByRole("textbox", { name: "Path" })).toBeDisabled(); + expect(changed).toHaveBeenLastCalledWith(legacy, [], true); + }); + + it("keeps existing standard policies valid and unchanged in compatibility-only GSI use", () => { + const changed = jest.fn(); + render(); + expect(screen.getByRole("button", { name: "Add full text path" })).toBeDisabled(); + expect(changed).toHaveBeenLastCalledWith(policy, [{ path: "/text" }, { path: "/other" }], true); + }); + + it("validates new empty paths and generates indexes without altering other policy fields", async () => { + const changed = jest.fn(); + render( + , + ); + fireEvent.click(screen.getByRole("button", { name: "Add full text path" })); + expect(changed.mock.calls[changed.mock.calls.length - 1][2]).toBe(false); + expect(await screen.findByText("Full text path should not be empty")).toBeVisible(); + const paths = screen.getAllByRole("textbox", { name: "Path" }); + fireEvent.change(paths[2], { target: { value: "added" } }); + const updated = changed.mock.calls[changed.mock.calls.length - 1]; + expect(updated[0].fullTextPaths[2]).toEqual({ path: "/added" }); + expect(updated[0].defaultSpec).toEqual(policy.defaultSpec); + expect(updated[1]).toEqual([{ path: "/text" }, { path: "/other" }, { path: "/added" }]); + expect(updated[2]).toBe(true); + }); }); diff --git a/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx b/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx index 16d7c62b1..5043029a4 100644 --- a/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx +++ b/src/Explorer/Controls/FullTextSeach/FullTextPoliciesComponent.tsx @@ -1,17 +1,42 @@ import { DefaultButton, Dropdown, + IButton, IDropdownOption, IDropdownStyles, IStyleFunctionOrObject, + ITextField, ITextFieldStyleProps, ITextFieldStyles, - Label, Stack, TextField, } from "@fluentui/react"; +import { + Accordion, + AccordionHeader, + AccordionItem, + AccordionPanel, + Button, + Checkbox, + makeStyles, + tokens, + useId, +} from "@fluentui/react-components"; +import { DeleteRegular } from "@fluentui/react-icons"; import { AccountOverride, FullTextIndex, FullTextPath, FullTextPolicy } from "Contracts/DataModels"; -import { CollapsibleSectionComponent } from "Explorer/Controls/CollapsiblePanel/CollapsibleSectionComponent"; +import { + fullTextLanguages, + getFullTextDefaultLanguage, + getStopwordValidationError, + inheritFullTextPathAnalysis, + isFullTextPathInherited, + isFullTextPolicyValid, + isSupportedFullTextPolicy, + setFullTextDefaultLanguage, +} from "Explorer/Controls/FullTextSeach/FullTextPolicyUtils"; +import { StopwordSettings } from "Explorer/Controls/FullTextSeach/StopwordSettings"; +import { CosmosFluentProvider } from "Explorer/Theme/ThemeUtil"; +import { t } from "Localization"; import * as React from "react"; import { isFullTextSearchPreviewFeaturesEnabled } from "Utils/CapabilityUtils"; @@ -26,22 +51,105 @@ export interface FullTextPoliciesComponentProps { onChangesDiscarded?: () => void; englishOnly?: boolean; targetAccountOverride?: AccountOverride; + allowStopwordCustomization?: boolean; + isEditing?: boolean; + fullTextIndexes?: FullTextIndex[]; } -export interface FullTextPolicyData { - path: string; - language: string; - pathError: string; -} +const emptyPolicy: FullTextPolicy = { defaultLanguage: "en-US", fullTextPaths: [] }; +const emptyIndexes: FullTextIndex[] = []; -const labelStyles = { - root: { - fontSize: 12, - color: "var(--colorNeutralForeground1)", +const useStyles = makeStyles({ + editor: { + minWidth: 0, + "& .fui-Field__label": { + paddingLeft: 0, + paddingRight: 0, + whiteSpace: "normal", + }, + "& .fui-Checkbox": { + maxWidth: "100%", + minWidth: 0, + }, + "& .fui-Checkbox__label": { + minWidth: 0, + whiteSpace: "normal", + overflowWrap: "anywhere", + }, + "& .fui-Textarea__textarea:disabled": { + color: tokens.colorNeutralForeground2, + }, + "& .fui-Dropdown__button:disabled": { + color: tokens.colorNeutralForeground2, + }, + "& .fui-Checkbox__input:disabled ~ .fui-Checkbox__label": { + color: tokens.colorNeutralForeground2, + }, }, -}; + pathsHeading: { + margin: 0, + paddingTop: tokens.spacingVerticalL, + paddingBottom: tokens.spacingVerticalS, + fontSize: tokens.fontSizeBase400, + lineHeight: tokens.lineHeightBase400, + fontWeight: tokens.fontWeightSemibold, + }, + pathSection: { + borderTop: `1px solid ${tokens.colorNeutralStroke2}`, + minWidth: 0, + }, + pathHeaderRow: { + display: "flex", + alignItems: "center", + columnGap: tokens.spacingHorizontalS, + }, + pathHeader: { + flex: 1, + minWidth: 0, + margin: 0, + }, + pathHeaderButton: { + paddingLeft: 0, + paddingRight: tokens.spacingHorizontalS, + minHeight: "56px", + minWidth: 0, + }, + pathIdentity: { + display: "grid", + rowGap: tokens.spacingVerticalXXS, + minWidth: 0, + textAlign: "left", + }, + pathName: { + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", + fontWeight: tokens.fontWeightSemibold, + }, + pathStatus: { + color: tokens.colorNeutralForeground2, + fontSize: tokens.fontSizeBase200, + lineHeight: tokens.lineHeightBase200, + fontWeight: tokens.fontWeightRegular, + whiteSpace: "normal", + }, + pathError: { + color: tokens.colorPaletteRedForeground1, + }, + pathPanel: { + margin: 0, + paddingBottom: tokens.spacingVerticalL, + }, + pathFields: { + minWidth: 0, + paddingLeft: tokens.spacingHorizontalL, + }, +}); const textFieldStyles: IStyleFunctionOrObject = { + subComponentStyles: { + label: { root: { color: "var(--colorNeutralForeground1)" } }, + }, fieldGroup: { height: 27, backgroundColor: "var(--colorNeutralBackground2)", @@ -54,6 +162,8 @@ const textFieldStyles: IStyleFunctionOrObject = { root: { - width: "40%", + width: "100%", + maxWidth: 240, marginTop: "10px", selectors: { "&:hover .ms-Dropdown-title": { @@ -208,194 +319,350 @@ export const FullTextPoliciesComponent: React.FunctionComponent { - const getFullTextPathError = (path: string, index?: number): string => { - let error = ""; - if (!path) { - error = "Full text path should not be empty"; - } - if ( - index >= 0 && - fullTextPathData?.find( - (fullTextPath: FullTextPolicyData, dataIndex: number) => dataIndex !== index && fullTextPath.path === path, - ) - ) { - error = "Full text path is already defined"; - } - return error; - }; - - const initializeData = (fullTextPolicy: FullTextPolicy): FullTextPolicyData[] => { - if (!fullTextPolicy) { - fullTextPolicy = { defaultLanguage: getFullTextLanguageOptions()[0].key as never, fullTextPaths: [] }; - } - - return fullTextPolicy.fullTextPaths.map((fullTextPath: FullTextPath) => ({ - ...fullTextPath, - pathError: getFullTextPathError(fullTextPath.path), - })); - }; - - const [fullTextPathData, setFullTextPathData] = React.useState(initializeData(fullTextPolicy)); - const [defaultLanguage, setDefaultLanguage] = React.useState( - fullTextPolicy - ? fullTextPolicy.defaultLanguage - : (getFullTextLanguageOptions(englishOnly, targetAccountOverride)[0].key as never), - ); + const styles = useStyles(); + const addPathButton = React.useRef(null); + const newPathInput = React.useRef(null); + const focusNewPath = React.useRef(false); + const pathDescriptionId = useId("full-text-path-description"); + const incomingPolicy = fullTextPolicy ?? emptyPolicy; + const [policy, setPolicy] = React.useState(incomingPolicy); + const initialPolicy = React.useRef(incomingPolicy); + const callbacks = React.useRef({ onFullTextPathChange, onChangesDiscarded }); + callbacks.current = { onFullTextPathChange, onChangesDiscarded }; + const canCustomize = allowStopwordCustomization && isFullTextSearchPreviewFeaturesEnabled(targetAccountOverride); + const showStopwords = canCustomize && (isEditing || policy.package === "standard"); + const supported = isSupportedFullTextPolicy(policy); + const hasCustomization = + policy.package === "standard" || + policy.defaultSpec !== undefined || + policy.fullTextPaths.some( + (path) => + path.stopWordListKind !== undefined || path.addStopWords !== undefined || path.removeStopWords !== undefined, + ); + const readOnly = !supported || (hasCustomization && !canCustomize); + const defaultsLocked = readOnly || (isEditing && fullTextIndexes.length > 0); + const defaultLanguage = getFullTextDefaultLanguage(policy); + const canToggleCustomization = + !isEditing && + initialPolicy.current.defaultSpec === undefined && + initialPolicy.current.package === undefined && + initialPolicy.current.fullTextPaths.every((path) => + Object.keys(path).every((key) => key === "path" || key === "language"), + ); React.useEffect(() => { - propagateData(); - }, [fullTextPathData, defaultLanguage]); + setPolicy(incomingPolicy); + }, [incomingPolicy]); React.useEffect(() => { if (discardChanges) { - setFullTextPathData(initializeData(fullTextPolicy)); - setDefaultLanguage(fullTextPolicy.defaultLanguage); - onChangesDiscarded(); + setPolicy(incomingPolicy); + callbacks.current.onChangesDiscarded?.(); } - }, [discardChanges]); + }, [discardChanges, incomingPolicy]); - const propagateData = () => { - const newFullTextPolicy: FullTextPolicy = { - defaultLanguage: defaultLanguage, - fullTextPaths: fullTextPathData.map((policy: FullTextPolicyData) => ({ - path: policy.path, - language: policy.language, - })), - }; - const fullTextIndexes: FullTextIndex[] = fullTextPathData.map((policy) => ({ - path: policy.path, + React.useEffect(() => { + if (focusNewPath.current) { + focusNewPath.current = false; + newPathInput.current?.focus(); + } + }, [policy.fullTextPaths.length]); + + React.useEffect(() => { + callbacks.current.onFullTextPathChange( + policy, + isEditing ? fullTextIndexes : policy.fullTextPaths.map(({ path }) => ({ path })), + readOnly || isFullTextPolicyValid(policy, isEditing ? fullTextIndexes : emptyIndexes), + ); + }, [policy, isEditing, fullTextIndexes, readOnly]); + + const updatePath = (index: number, update: (path: FullTextPath) => FullTextPath): void => + setPolicy((current) => ({ + ...current, + fullTextPaths: current.fullTextPaths.map((path, pathIndex) => (pathIndex === index ? update(path) : path)), })); - const validationPassed = fullTextPathData.every((policy: FullTextPolicyData) => policy.pathError === ""); - onFullTextPathChange(newFullTextPolicy, fullTextIndexes, validationPassed); - }; - const onFullTextPathValueChange = (index: number, event: React.ChangeEvent) => { - const value = event.target.value.trim(); - const fullTextPaths = [...fullTextPathData]; - if (!fullTextPaths[index]?.path && !value.startsWith("/")) { - fullTextPaths[index].path = "/" + value; - } else { - fullTextPaths[index].path = value; + const languageOptions = (currentLanguage: string): IDropdownOption[] => { + const options = getFullTextLanguageOptions(englishOnly, targetAccountOverride); + if (!options.some((option) => option.key === currentLanguage)) { + options.push({ key: currentLanguage, text: currentLanguage, disabled: true }); } - fullTextPaths[index].pathError = getFullTextPathError(value, index); - setFullTextPathData(fullTextPaths); - }; - - const onFullTextPathPolicyChange = (index: number, option: IDropdownOption): void => { - const policies = [...fullTextPathData]; - policies[index].language = option.key as never; - setFullTextPathData(policies); - }; - - const onAdd = () => { - setFullTextPathData([ - ...fullTextPathData, - { - path: "", - language: defaultLanguage, - pathError: getFullTextPathError(""), - }, - ]); - }; - - const onDelete = (index: number) => { - const policies = fullTextPathData.filter((_uniqueKey, j) => index !== j); - setFullTextPathData(policies); + return options; }; return ( - - - - , option: IDropdownOption) => - setDefaultLanguage(option.key as never) - } - > - - {fullTextPathData && - fullTextPathData.length > 0 && - fullTextPathData.map((fullTextPolicy: FullTextPolicyData, index: number) => ( - onDelete(index)} - > - - + + {readOnly && ( +
{t(supported ? "fullTextPolicy.capabilityRequired" : "fullTextPolicy.unsupported")}
+ )} + + { + if (option && typeof option.key === "string") { + setPolicy((current) => setFullTextDefaultLanguage(current, option.key.toString())); + } + }} + > + + {canCustomize && canToggleCustomization && ( + { + if (data.checked === true) { + setPolicy((current) => ({ + ...current, + package: "standard", + defaultSpec: { + language: getFullTextDefaultLanguage(current), + stopWordListKind: "basic", + tokenizer: "word", + filters: ["lowercase", "stop"], }, - }} - > - - - ) => onFullTextPathValueChange(index, event)} - value={fullTextPolicy.path || ""} - errorMessage={fullTextPolicy.pathError} - /> - - - - , option: IDropdownOption) => - onFullTextPathPolicyChange(index, option) - } - > - - - - - ))} - - Add full text path - - + fullTextPaths: current.fullTextPaths.map((path) => + path.language === getFullTextDefaultLanguage(current) && + Object.keys(path).every((key) => key === "path" || key === "language") + ? { path: path.path } + : path, + ), + })); + } else { + setPolicy((current) => { + const language = getFullTextDefaultLanguage(current); + const next = { + ...current, + defaultLanguage: language, + fullTextPaths: current.fullTextPaths.map((path) => ({ + ...inheritFullTextPathAnalysis(path), + language: path.language ?? language, + })), + }; + delete next.package; + delete next.defaultSpec; + return next; + }); + } + }} + /> + )} + {showStopwords && ( + <> + {!isEditing &&
{t("fullTextPolicy.customizeDescription")}
} + {defaultsLocked && !readOnly &&
{t("fullTextPolicy.defaultLocked")}
} + setPolicy((current) => ({ ...current, defaultSpec }))} + /> + + )} +

+ {t("fullTextPolicy.pathsHeading", { count: policy.fullTextPaths.length })} +

+ {policy.fullTextPaths.length === 0 &&
{t("fullTextPolicy.emptyPaths")}
} + {policy.fullTextPaths.map((path, index) => { + const indexed = isEditing && fullTextIndexes.some((entry) => entry.path === path.path); + const pathLocked = readOnly || indexed; + const inherited = isFullTextPathInherited(path); + const pathError = !path.path.trim() + ? t("fullTextPolicy.pathRequired") + : policy.fullTextPaths.some((entry, entryIndex) => entryIndex !== index && entry.path === path.path) + ? t("fullTextPolicy.pathDuplicate") + : undefined; + const needsAttention = !pathLocked && (pathError || getStopwordValidationError(path, policy.package)); + const analysisMode = t(inherited ? "fullTextPolicy.inheritsDefaults" : "fullTextPolicy.overridesDefaults"); + const pathSummary = indexed + ? t("fullTextPolicy.indexedPathSummary", { mode: analysisMode }) + : readOnly + ? t("fullTextPolicy.readOnlyPathSummary", { mode: analysisMode }) + : analysisMode; + return ( + + +
+ + + + {path.path || t("fullTextPolicy.newPath")} + + + {pathSummary} + {needsAttention && ( + + {" - "} + {t("fullTextPolicy.needsAttention")} + + )} + + + + {!pathLocked && ( +
+ + + {pathLocked && !readOnly &&
{t("fullTextPolicy.pathLocked")}
} + + + updatePath(index, (current) => { + const next = (value ?? "").trim(); + return { + ...current, + path: !current.path && next.length > 0 && !next.startsWith("/") ? `/${next}` : next, + }; + }) + } + value={path.path} + errorMessage={pathError} + /> + + {showStopwords && ( + <> + + updatePath(index, (current) => + data.checked === true + ? inheritFullTextPathAnalysis(current) + : { ...current, language: defaultLanguage }, + ) + } + /> +
+ {inherited + ? t("fullTextPolicy.inheritedLanguageDescription", { + language: + languageOptions(defaultLanguage).find((option) => option.key === defaultLanguage) + ?.text ?? defaultLanguage, + }) + : t("fullTextPolicy.overrideDescription")} +
+ + )} + {(!inherited || !showStopwords) && ( + + { + if (option && typeof option.key === "string") { + updatePath(index, (current) => ({ ...current, language: option.key.toString() })); + } + }} + > + + )} + {showStopwords && !inherited && ( + updatePath(index, (current) => ({ ...spec, path: current.path }))} + /> + )} + + + + + ); + })} + { + focusNewPath.current = true; + setPolicy((current) => ({ + ...current, + fullTextPaths: [ + ...current.fullTextPaths, + current.package === "standard" + ? { path: "" } + : { path: "", language: getFullTextDefaultLanguage(current) }, + ], + })); + }} + > + {t("fullTextPolicy.addPath")} + + + ); }; @@ -405,28 +672,7 @@ export const getFullTextLanguageOptions = ( ): IDropdownOption[] => { const multiLanguageSupportEnabled: boolean = isFullTextSearchPreviewFeaturesEnabled(targetAccountOverride) && !englishOnly; - const fullTextLanguageOptions: IDropdownOption[] = [ - { - key: "en-US", - text: "English (US)", - }, - ...(multiLanguageSupportEnabled - ? [ - { - key: "fr-FR", - text: "French", - }, - { - key: "de-DE", - text: "German", - }, - { - key: "es-ES", - text: "Spanish", - }, - ] - : []), - ]; - - return fullTextLanguageOptions; + return fullTextLanguages + .filter((language) => language.value === "en-US" || multiLanguageSupportEnabled) + .map(({ value, label }) => ({ key: value, text: t(`fullTextPolicy.${label}`) })); }; diff --git a/src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.test.ts b/src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.test.ts new file mode 100644 index 000000000..a5ba087b8 --- /dev/null +++ b/src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.test.ts @@ -0,0 +1,324 @@ +import { FullTextAnalysisSpec, FullTextPolicy } from "Contracts/DataModels"; +import { + fullTextLanguages, + getFullTextDefaultLanguage, + getStopwordPresets, + getStopwordValidationError, + inheritFullTextPathAnalysis, + isFullTextPathInherited, + isFullTextPolicyValid, + isSupportedFullTextPolicy, + setFullTextDefaultLanguage, +} from "./FullTextPolicyUtils"; + +describe("full-text policy helpers", () => { + const policy: FullTextPolicy = { + package: "standard", + defaultSpec: { language: "en-US", stopWordListKind: "basic", filters: ["lowercase", "stop"] }, + fullTextPaths: [{ path: "/text" }], + }; + + it("lists exactly the seven verified locale choices", () => { + expect(fullTextLanguages.map(({ value }) => value)).toEqual([ + "en-US", + "fr-FR", + "de-DE", + "es-ES", + "it-IT", + "pt-PT", + "pt-BR", + ]); + }); + + describe("complete supported language and stopword policy matrix", () => { + it.each(fullTextLanguages)("$value and $legacyId expose only their supported presets", ({ value, legacyId }) => { + for (const language of [value, legacyId]) { + expect(getStopwordPresets("standard", language)).toEqual(["none", "basic"]); + const expectedLegacy = value === "en-US" ? ["none", "extended"] : ["none"]; + expect(getStopwordPresets("legacy", language)).toEqual(expectedLegacy); + expect(getStopwordPresets(undefined, language)).toEqual(expectedLegacy); + for (const packageName of ["standard", "legacy", undefined]) { + for (const preset of ["none", "basic", "extended"]) { + const valid = packageName === "standard" ? preset !== "extended" : expectedLegacy.includes(preset); + const spec = { + language, + stopWordListKind: preset, + addStopWords: ["catalog", "Catalog", "catalog", "caf\u00e9"], + }; + expect(getStopwordValidationError(spec, packageName)).toBe(valid ? undefined : "invalidPreset"); + } + } + } + }); + + const transitions = fullTextLanguages.flatMap((from) => + fullTextLanguages.map((to) => ({ from: from.value, to: to.value })), + ); + it.each(transitions)("$from -> $to preserves explicit path overrides and unknown fields", ({ from, to }) => { + const paths = [ + { path: "/inherited" }, + { path: "/override", language: from, stopWordListKind: "none", addStopWords: ["Original", "original"] }, + ]; + for (const shape of ["legacy", "modern", "both"]) { + const original: FullTextPolicy = { + ...(shape !== "modern" ? { defaultLanguage: from } : {}), + ...(shape !== "legacy" + ? { + package: "standard", + defaultSpec: { + language: from, + stopWordListKind: "none", + addStopWords: ["keep", "Keep", "keep"], + tokenizer: "word", + filters: ["lowercase", "stop"], + future: { preserved: true }, + }, + } + : {}), + fullTextPaths: paths, + futurePolicy: { preserved: true }, + }; + const changed = setFullTextDefaultLanguage(original, to); + expect(getFullTextDefaultLanguage(changed)).toBe(to); + expect(changed.fullTextPaths).toEqual(paths); + expect(changed.futurePolicy).toEqual({ preserved: true }); + expect(getFullTextDefaultLanguage(original)).toBe(from); + expect(changed.defaultLanguage).toBe(shape === "modern" ? undefined : to); + expect(changed.defaultSpec).toEqual(shape === "legacy" ? undefined : { ...original.defaultSpec, language: to }); + expect(isFullTextPolicyValid(changed)).toBe(true); + } + }); + + const listCases = fullTextLanguages.flatMap(({ value }) => + [0, 1, 5, 20, 100, 1001].map((count) => ({ language: value, count })), + ); + it.each(listCases)("$language preserves $count custom words without an invented cap", ({ language, count }) => { + const words = Array.from( + { length: count }, + (_, index) => ["catalog", "Catalog", "caf\u00e9", "catalog"][index % 4], + ); + for (const field of ["addStopWords", "removeStopWords"] as const) { + const spec = { language, stopWordListKind: "none", [field]: words }; + expect(getStopwordValidationError(spec, "standard")).toBeUndefined(); + expect(getStopwordValidationError(spec, "legacy")).toBeUndefined(); + expect(spec[field]).toBe(words); + } + }); + + const forbidden = [ + "two words", + "a\tb", + "a\nb", + "a\rb", + "a\u00a0b", + "a\u0000b", + "a\u007fb", + "a-b", + "can't", + "a\u2019b", + "a\u2014b", + "a/b", + "a,b", + "a.b", + "C++", + "$value", + "a_b", + "a:b", + ]; + it.each(fullTextLanguages)("$value rejects forbidden characters anywhere in either list", ({ value: language }) => { + for (const field of ["addStopWords", "removeStopWords"] as const) { + for (const invalid of forbidden) { + const words = Array.from({ length: 20 }, () => "valid"); + words[18] = invalid; + expect(getStopwordValidationError({ language, stopWordListKind: "none", [field]: words }, "standard")).toBe( + "invalidWord", + ); + } + expect(getStopwordValidationError({ language, [field]: [] }, "standard")).toBe("presetRequired"); + expect(getStopwordValidationError({ stopWordListKind: "none", [field]: ["valid"] }, "standard")).toBe( + "languageRequired", + ); + } + }); + + it.each(fullTextLanguages)( + "$value removes all supported analysis overrides when inheriting", + ({ value: language }) => { + const path = { + path: "/text", + language, + stopWordListKind: "none", + addStopWords: ["one", "two"], + removeStopWords: ["three"], + tokenizer: "word", + filters: ["lowercase", "stop"], + future: { unchanged: true }, + }; + expect(inheritFullTextPathAnalysis(path)).toEqual({ + path: "/text", + future: { unchanged: true }, + }); + expect(path.addStopWords).toEqual(["one", "two"]); + }, + ); + }); + + it("resolves defaults without changing the wire policy", () => { + expect(getFullTextDefaultLanguage({ fullTextPaths: [] })).toBe("en-US"); + expect(getFullTextDefaultLanguage({ defaultLanguage: "fr-FR", fullTextPaths: [] })).toBe("fr-FR"); + expect(getFullTextDefaultLanguage({ ...policy, defaultLanguage: "fr-FR" })).toBe("en-US"); + }); + + it.each<[string | undefined, string, string[]]>([ + [undefined, "en-US", ["none", "extended"]], + ["legacy", "1033", ["none", "extended"]], + ["legacy", "fr-FR", ["none"]], + ["standard", "en-US", ["none", "basic"]], + ["standard", "fr-FR", ["none", "basic"]], + ["future", "en-US", []], + ])("uses package/language-specific presets for %s / %s", (packageName, language, expected) => { + expect(getStopwordPresets(packageName, language)).toEqual(expected); + }); + + it("preserves unknown fields and does not introduce a legacy defaultLanguage into a modern policy", () => { + const original: FullTextPolicy = { + ...policy, + futurePolicySetting: { enabled: true }, + defaultSpec: { ...policy.defaultSpec, futureAnalysisSetting: ["unchanged"] }, + }; + const changed = setFullTextDefaultLanguage(original, "fr-FR"); + expect(changed).toEqual({ + ...original, + defaultSpec: { ...original.defaultSpec, language: "fr-FR" }, + }); + expect(changed.defaultLanguage).toBeUndefined(); + expect(original.defaultSpec?.language).toBe("en-US"); + }); + + it("keeps both language fields consistent when the service returned both", () => { + const changed = setFullTextDefaultLanguage({ ...policy, defaultLanguage: "en-US" }, "de-DE"); + expect(changed.defaultLanguage).toBe("de-DE"); + expect(changed.defaultSpec?.language).toBe("de-DE"); + }); + + it("resets all supported analysis overrides while preserving unknown fields", () => { + const path = { + path: "/text", + language: "fr-FR", + stopWordListKind: "none", + addStopWords: ["cosmos"], + removeStopWords: ["the"], + tokenizer: "word", + filters: ["lowercase", "ascii", "stem", "stop"], + futureSetting: { value: 1 }, + }; + expect(inheritFullTextPathAnalysis(path)).toEqual({ + path: "/text", + futureSetting: { value: 1 }, + }); + expect(path.addStopWords).toEqual(["cosmos"]); + }); + + it("treats a path as inherited only when every supported analysis field is absent", () => { + expect(isFullTextPathInherited({ path: "/text", futureSetting: { preserved: true } })).toBe(true); + const overrides: FullTextAnalysisSpec[] = [ + { language: "en-US" }, + { stopWordListKind: "none" }, + { addStopWords: [] }, + { removeStopWords: [] }, + { tokenizer: "word" }, + { filters: [] }, + ]; + for (const analysis of overrides) { + expect(isFullTextPathInherited({ path: "/text", ...analysis })).toBe(false); + } + }); + + it("requires explicit language and preset when custom lists are supplied, including empty lists", () => { + expect(getStopwordValidationError({ addStopWords: [] }, "standard")).toBe("languageRequired"); + expect(getStopwordValidationError({ language: "en-US", removeStopWords: [] }, "standard")).toBe("presetRequired"); + expect( + getStopwordValidationError({ language: "en-US", stopWordListKind: "none", addStopWords: [] }, "standard"), + ).toBeUndefined(); + expect(getStopwordValidationError({ language: "fr-FR", stopWordListKind: "extended" }, "legacy")).toBe( + "invalidPreset", + ); + }); + + it.each(["two words", "word-word", "can't", "C++", "$money", "a\tb", "a\u0000b", "a\u2014b"])( + "rejects a custom word containing a forbidden character: %s", + (word) => { + expect( + getStopwordValidationError({ language: "en-US", stopWordListKind: "none", addStopWords: [word] }, "standard"), + ).toBe("invalidWord"); + }, + ); + + it("does not silently normalize or impose an invented count limit on custom words", () => { + const words = ["Cosmos", "cosmos", "caf\u00e9", "", ...Array.from({ length: 1001 }, (_, i) => `word${i}`)]; + const spec = { language: "en-US", stopWordListKind: "basic", addStopWords: words, removeStopWords: ["Cosmos"] }; + expect(getStopwordValidationError(spec, "standard")).toBeUndefined(); + expect(spec.addStopWords).toBe(words); + expect(spec.removeStopWords).toEqual(["Cosmos"]); + }); + + it("validates paths and their overrides separately from container defaults", () => { + expect(isFullTextPolicyValid(policy)).toBe(true); + expect(isFullTextPolicyValid({ ...policy, fullTextPaths: [{ path: "" }] })).toBe(false); + expect(isFullTextPolicyValid({ ...policy, fullTextPaths: [{ path: "/text" }, { path: "/text" }] })).toBe(false); + expect( + isFullTextPolicyValid({ + ...policy, + fullTextPaths: [{ path: "/text", language: "en-US", addStopWords: ["extra"] }], + }), + ).toBe(false); + }); + + it("ignores only locked analysis while continuing to validate editable paths", () => { + const persisted: FullTextPolicy = { + ...policy, + defaultSpec: { ...policy.defaultSpec, stopWordListKind: "extended" }, + fullTextPaths: [{ path: "/text", language: "en-US", stopWordListKind: "extended" }], + }; + expect(isFullTextPolicyValid(persisted)).toBe(false); + expect(isFullTextPolicyValid(persisted, [{ path: "/text" }])).toBe(true); + expect( + isFullTextPolicyValid({ ...persisted, fullTextPaths: [...persisted.fullTextPaths, { path: "/other" }] }, [ + { path: "/text" }, + ]), + ).toBe(true); + expect( + isFullTextPolicyValid({ ...persisted, fullTextPaths: [...persisted.fullTextPaths, { path: "" }] }, [ + { path: "/text" }, + ]), + ).toBe(false); + expect( + isFullTextPolicyValid( + { + ...persisted, + fullTextPaths: [ + ...persisted.fullTextPaths, + { path: "/other", language: "en-US", stopWordListKind: "basic", addStopWords: ["two words"] }, + ], + }, + [{ path: "/text" }], + ), + ).toBe(false); + expect( + isFullTextPolicyValid({ ...persisted, fullTextPaths: [...persisted.fullTextPaths, { path: "/text" }] }, [ + { path: "/text" }, + ]), + ).toBe(false); + }); + + it.each([ + { ...policy, package: "future" }, + { ...policy, defaultSpec: { language: "en-US", stopWordListKind: "future" } }, + { ...policy, defaultSpec: { language: "japanese" } }, + { ...policy, defaultSpec: { language: "en-US", tokenizer: "future" } }, + { ...policy, defaultSpec: { language: "en-US", filters: ["future"] } }, + ])("marks future analysis read-only without blocking an unrelated save", (futurePolicy) => { + expect(isSupportedFullTextPolicy(futurePolicy)).toBe(false); + expect(isFullTextPolicyValid(futurePolicy)).toBe(true); + }); +}); diff --git a/src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.ts b/src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.ts new file mode 100644 index 000000000..7aafc3f94 --- /dev/null +++ b/src/Explorer/Controls/FullTextSeach/FullTextPolicyUtils.ts @@ -0,0 +1,121 @@ +import { FullTextAnalysisSpec, FullTextIndex, FullTextPath, FullTextPolicy } from "Contracts/DataModels"; + +export type StopwordValidationError = "languageRequired" | "presetRequired" | "invalidPreset" | "invalidWord"; + +export const fullTextLanguages = [ + { value: "en-US", legacyId: "1033", label: "english" }, + { value: "fr-FR", legacyId: "1036", label: "french" }, + { value: "de-DE", legacyId: "1031", label: "german" }, + { value: "es-ES", legacyId: "3082", label: "spanish" }, + { value: "it-IT", legacyId: "1040", label: "italian" }, + { value: "pt-PT", legacyId: "2070", label: "portuguesePortugal" }, + { value: "pt-BR", legacyId: "1046", label: "portugueseBrazil" }, +] as const; + +export const getFullTextDefaultLanguage = (policy: FullTextPolicy): string => + policy.defaultSpec?.language ?? policy.defaultLanguage ?? "en-US"; + +export const getStopwordPresets = (packageName: string | undefined, language: string): string[] => { + if (packageName === "standard") { + return ["none", "basic"]; + } + if (packageName === undefined || packageName === "legacy") { + return language === "en-US" || language === "1033" ? ["none", "extended"] : ["none"]; + } + return []; +}; + +export const isSupportedFullTextPolicy = (policy: FullTextPolicy): boolean => { + if (policy.package !== undefined && policy.package !== "legacy" && policy.package !== "standard") { + return false; + } + const specs: (FullTextAnalysisSpec | undefined)[] = [ + { language: policy.defaultLanguage }, + policy.defaultSpec, + ...policy.fullTextPaths, + ]; + return specs.every( + (spec) => + !spec || + ((!spec.language || + fullTextLanguages.some(({ value, legacyId }) => value === spec.language || legacyId === spec.language)) && + (spec.stopWordListKind === undefined || ["none", "basic", "extended"].includes(spec.stopWordListKind)) && + (spec.tokenizer === undefined || spec.tokenizer === "word") && + (spec.filters === undefined || + spec.filters.every((filter) => ["lowercase", "ascii", "stop", "stem"].includes(filter)))), + ); +}; + +export const getStopwordValidationError = ( + spec: FullTextAnalysisSpec | undefined, + packageName: string | undefined, +): StopwordValidationError | undefined => { + if (!spec) { + return undefined; + } + const hasWords = spec.addStopWords !== undefined || spec.removeStopWords !== undefined; + if ((spec.language !== undefined || spec.stopWordListKind !== undefined || hasWords) && !spec.language) { + return "languageRequired"; + } + if (hasWords && spec.stopWordListKind === undefined) { + return "presetRequired"; + } + if ( + spec.stopWordListKind !== undefined && + !getStopwordPresets(packageName, spec.language ?? "en-US").includes(spec.stopWordListKind) + ) { + return "invalidPreset"; + } + // Mirror the server's whitespace/control/punctuation checks without normalizing customer words. + const invalidWord = /[\s\p{Cc}\x21-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e\p{P}]/u; + if ([...(spec.addStopWords ?? []), ...(spec.removeStopWords ?? [])].some((word) => invalidWord.test(word))) { + return "invalidWord"; + } + return undefined; +}; + +export const isFullTextPolicyValid = ( + policy: FullTextPolicy, + lockedIndexes: readonly FullTextIndex[] = [], +): boolean => { + // Future policies are read-only in this editor and must not block unrelated settings saves. + if (!isSupportedFullTextPolicy(policy)) { + return true; + } + const lockedPaths = new Set(lockedIndexes.map(({ path }) => path)); + const paths = policy.fullTextPaths.map(({ path }) => path); + return ( + paths.every((path) => path.trim().length > 0) && + new Set(paths).size === paths.length && + (lockedPaths.size > 0 || getStopwordValidationError(policy.defaultSpec, policy.package) === undefined) && + policy.fullTextPaths.every( + (path) => lockedPaths.has(path.path) || getStopwordValidationError(path, policy.package) === undefined, + ) + ); +}; + +export const setFullTextDefaultLanguage = (policy: FullTextPolicy, language: string): FullTextPolicy => ({ + ...policy, + ...(!policy.defaultSpec || policy.defaultLanguage !== undefined ? { defaultLanguage: language } : {}), + ...(policy.defaultSpec ? { defaultSpec: { ...policy.defaultSpec, language } } : {}), +}); + +const analysisFields = [ + "language", + "stopWordListKind", + "addStopWords", + "removeStopWords", + "tokenizer", + "filters", +] as const; + +export const isFullTextPathInherited = (path: FullTextPath): boolean => + analysisFields.every((field) => path[field] === undefined); + +export const inheritFullTextPathAnalysis = (path: FullTextPath): FullTextPath => { + const result = { ...path }; + for (const field of analysisFields) { + delete result[field]; + } + return result; +}; diff --git a/src/Explorer/Controls/FullTextSeach/StopwordSettings.test.tsx b/src/Explorer/Controls/FullTextSeach/StopwordSettings.test.tsx new file mode 100644 index 000000000..84bd85bf7 --- /dev/null +++ b/src/Explorer/Controls/FullTextSeach/StopwordSettings.test.tsx @@ -0,0 +1,241 @@ +import "@testing-library/jest-dom"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { FullTextAnalysisSpec } from "Contracts/DataModels"; +import React from "react"; +import { StopwordSettings } from "./StopwordSettings"; +import { fullTextLanguages } from "./FullTextPolicyUtils"; + +const spec: FullTextAnalysisSpec = { + language: "en-US", + stopWordListKind: "basic", + tokenizer: "word", + filters: ["lowercase", "stop"], + addStopWords: ["cosmos"], + removeStopWords: ["the"], +}; + +describe("stopword design guidance", () => { + it("explains default scope without changing the selected policy", () => { + const onChange = jest.fn(); + render( + , + ); + expect(screen.getByText("Used by full-text paths that inherit the container settings.")).toBeVisible(); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).toHaveValue("cosmos"); + expect(onChange).not.toHaveBeenCalled(); + }); + + describe("multiword editor language matrix", () => { + it.each(fullTextLanguages)("$value edits either list losslessly at every tested size", ({ value: language }) => { + const changed = jest.fn(); + const Harness = () => { + const [current, setCurrent] = React.useState({ + ...spec, + language, + future: { retained: true }, + }); + return ( + { + changed(next); + setCurrent(next); + }} + /> + ); + }; + render(); + const added = screen.getByRole("textbox", { name: "Additional stopwords" }); + const kept = screen.getByRole("textbox", { name: "Words to keep" }); + for (const count of [0, 1, 5, 20, 100, 1001]) { + const words = Array.from( + { length: count }, + (_, index) => ["catalog", "Catalog", "caf\u00e9", "catalog"][index % 4], + ); + fireEvent.change(added, { target: { value: words.join("\r\n") } }); + expect(added).toHaveValue(words.join("\n")); + expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ language, addStopWords: words })); + fireEvent.change(kept, { target: { value: words.join("\n") } }); + expect(kept).toHaveValue(words.join("\n")); + expect(changed).toHaveBeenLastCalledWith( + expect.objectContaining({ + language, + addStopWords: words, + removeStopWords: words, + future: { retained: true }, + }), + ); + } + fireEvent.click(screen.getByRole("checkbox", { name: "Enable stopword filtering" })); + expect(added).toBeDisabled(); + expect(kept).toBeDisabled(); + expect(changed).toHaveBeenLastCalledWith(expect.objectContaining({ filters: ["lowercase"] })); + fireEvent.click(screen.getByRole("checkbox", { name: "Enable stopword filtering" })); + expect(added).toBeEnabled(); + expect(kept).toBeEnabled(); + expect(added).toHaveValue( + Array.from({ length: 1001 }, (_, index) => ["catalog", "Catalog", "caf\u00e9", "catalog"][index % 4]).join( + "\n", + ), + ); + }); + + it.each(fullTextLanguages)("$value associates a late error with only the offending list", ({ value: language }) => { + const words = Array.from({ length: 20 }, () => "valid"); + words[18] = "invalid phrase"; + const props = { label: "Words", language, packageName: "standard", disabled: false, onChange: jest.fn() }; + const { rerender } = render(); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).toHaveAttribute("aria-invalid", "true"); + expect(screen.getByRole("textbox", { name: "Words to keep" })).not.toHaveAttribute("aria-invalid", "true"); + rerender(); + expect(screen.getByRole("textbox", { name: "Words to keep" })).toHaveAttribute("aria-invalid", "true"); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).not.toHaveAttribute("aria-invalid", "true"); + rerender( + , + ); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).toHaveValue(words.join("\n")); + }); + }); + + it("explains why service-default word fields are disabled without selecting a preset", () => { + const props = { + label: "Default stopwords", + language: "en-US", + packageName: "standard", + disabled: false, + onChange: jest.fn(), + }; + const { rerender } = render( + , + ); + expect(screen.getByRole("combobox", { name: "Stopword list" })).toHaveAccessibleDescription( + "Select a stopword list to customize the words below.", + ); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).toBeDisabled(); + expect(props.onChange).not.toHaveBeenCalled(); + rerender(); + expect(screen.queryByText("Select a stopword list to customize the words below.")).not.toBeInTheDocument(); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).toBeEnabled(); + }); + + it.each<[string[], string[], boolean, boolean]>([ + [["invalid phrase"], ["the"], true, false], + [["cosmos"], ["invalid phrase"], false, true], + [["bad!"], ["bad?"], true, true], + [["cosmos"], ["the"], false, false], + ])( + "associates validation with the exact word list: %j / %j", + (addStopWords, removeStopWords, addedInvalid, removedInvalid) => { + render( + , + ); + const added = screen.getByRole("textbox", { name: "Additional stopwords" }); + const removed = screen.getByRole("textbox", { name: "Words to keep" }); + expect(added.getAttribute("aria-invalid") === "true").toBe(addedInvalid); + expect(removed.getAttribute("aria-invalid") === "true").toBe(removedInvalid); + expect(added).toHaveAccessibleDescription( + addedInvalid + ? /Stopwords cannot contain whitespace, punctuation, or control characters\./ + : "Also ignore these words. Enter one word per line. Words cannot contain whitespace, punctuation, or control characters.", + ); + expect(removed).toHaveAccessibleDescription( + removedInvalid + ? /Stopwords cannot contain whitespace, punctuation, or control characters\./ + : "Keep these words even if they are in the stopword list. Enter one word per line. Words cannot contain whitespace, punctuation, or control characters.", + ); + expect(screen.queryAllByRole("alert")).toHaveLength(Number(addedInvalid) + Number(removedInvalid)); + }, + ); + + it.each([ + ["none", "No built-in stopwords. Only your additional stopwords are ignored."], + ["basic", "Uses the built-in stopword list for this language."], + ["extended", "Uses the legacy English stopword list."], + ])("explains the %s preset without changing the policy", (stopWordListKind, description) => { + const onChange = jest.fn(); + render( + , + ); + expect(screen.getByRole("combobox", { name: "Stopword list" })).toHaveAccessibleDescription(description); + expect( + screen.getAllByText( + "Enter one word per line. Words cannot contain whitespace, punctuation, or control characters.", + ), + ).toHaveLength(1); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("does not mark locked values invalid or change serialization", () => { + const onChange = jest.fn(); + const props = { label: "Default stopwords", language: "en-US", packageName: "standard", onChange }; + const { rerender } = render( + , + ); + expect(screen.getByRole("textbox", { name: "Additional stopwords" })).toBeDisabled(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + rerender(); + fireEvent.change(screen.getByRole("textbox", { name: "Additional stopwords" }), { + target: { value: "Cosmos\ncosmos\ncosmos" }, + }); + expect(onChange).toHaveBeenCalledWith({ ...spec, addStopWords: ["Cosmos", "cosmos", "cosmos"] }); + }); + + it("keeps meanings but hides editing instructions for locked values", () => { + const props = { + label: "Default stopwords", + language: "en-US", + packageName: "standard", + spec, + onChange: jest.fn(), + }; + const { rerender } = render(); + const added = screen.getByRole("textbox", { name: "Additional stopwords" }); + expect(added).toHaveValue("cosmos"); + expect(added).toBeDisabled(); + expect(added).toHaveAccessibleDescription("Also ignore these words."); + expect(screen.getByRole("textbox", { name: "Words to keep" })).toHaveAccessibleDescription( + "Keep these words even if they are in the stopword list.", + ); + expect( + screen.queryByText( + "Enter one word per line. Words cannot contain whitespace, punctuation, or control characters.", + ), + ).not.toBeInTheDocument(); + rerender(); + expect(added).toHaveAccessibleDescription( + "Also ignore these words. Enter one word per line. Words cannot contain whitespace, punctuation, or control characters.", + ); + expect(props.onChange).not.toHaveBeenCalled(); + }); +}); diff --git a/src/Explorer/Controls/FullTextSeach/StopwordSettings.tsx b/src/Explorer/Controls/FullTextSeach/StopwordSettings.tsx new file mode 100644 index 000000000..10a4445e9 --- /dev/null +++ b/src/Explorer/Controls/FullTextSeach/StopwordSettings.tsx @@ -0,0 +1,210 @@ +import { Checkbox, Dropdown, Field, makeStyles, Option, Textarea, tokens, useId } from "@fluentui/react-components"; +import { FullTextAnalysisSpec } from "Contracts/DataModels"; +import { getStopwordPresets, getStopwordValidationError } from "Explorer/Controls/FullTextSeach/FullTextPolicyUtils"; +import { t } from "Localization"; +import React from "react"; + +interface StopwordSettingsProps { + label: string; + description?: string; + language: string; + packageName: string | undefined; + spec: FullTextAnalysisSpec; + inheritedFilters?: string[]; + disabled: boolean; + onChange: (spec: FullTextAnalysisSpec) => void; +} + +const useStyles = makeStyles({ + root: { + minWidth: 0, + paddingTop: tokens.spacingVerticalS, + display: "grid", + rowGap: tokens.spacingVerticalM, + }, + heading: { + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase300, + lineHeight: tokens.lineHeightBase300, + }, + introduction: { + display: "grid", + rowGap: tokens.spacingVerticalXS, + }, + wordLists: { + display: "grid", + alignItems: "start", + gridTemplateColumns: "repeat(auto-fit, minmax(min(100%, 200px), 1fr))", + gap: tokens.spacingHorizontalL, + minWidth: 0, + maxWidth: "496px", + }, + wordField: { + minWidth: 0, + maxWidth: "240px", + }, + hint: { + color: tokens.colorNeutralForeground2, + fontSize: tokens.fontSizeBase200, + lineHeight: tokens.lineHeightBase200, + }, +}); + +export const StopwordSettings = ({ + label, + description, + language, + packageName, + spec, + inheritedFilters, + disabled, + onChange, +}: StopwordSettingsProps): JSX.Element => { + const styles = useStyles(); + const formatHintId = useId("stopword-format"); + const error = getStopwordValidationError(spec, packageName); + const addedWordsInvalid = + !disabled && getStopwordValidationError({ ...spec, removeStopWords: undefined }, packageName) === "invalidWord"; + const removedWordsInvalid = + !disabled && getStopwordValidationError({ ...spec, addStopWords: undefined }, packageName) === "invalidWord"; + const effectiveFilters = spec.filters ?? (spec.tokenizer === undefined ? inheritedFilters : undefined) ?? []; + const filteringEnabled = packageName !== "standard" || effectiveFilters.includes("stop"); + const presetLabel = (preset: string): string => { + switch (preset) { + case "none": + return t("fullTextPolicy.none"); + case "basic": + return t("fullTextPolicy.basic"); + case "extended": + return t("fullTextPolicy.extended"); + default: + return preset; + } + }; + const presetDescription = (): string | undefined => { + if (!filteringEnabled) { + return undefined; + } + switch (spec.stopWordListKind) { + case "none": + return t("fullTextPolicy.noneDescription"); + case "basic": + return t("fullTextPolicy.basicDescription"); + case "extended": + return t("fullTextPolicy.extendedDescription"); + default: + return !disabled ? t("fullTextPolicy.selectListHint") : undefined; + } + }; + const updateWords = (field: "addStopWords" | "removeStopWords", value: string): void => + onChange({ ...spec, language, [field]: value === "" ? [] : value.split(/\r?\n/) }); + + return ( +
+
+
{label}
+ {description &&
{description}
} +
+ {packageName === "standard" && ( + + onChange({ + ...spec, + language, + tokenizer: spec.tokenizer ?? "word", + filters: + data.checked === true + ? [...effectiveFilters.filter((filter) => filter !== "stop"), "stop"] + : effectiveFilters.filter((filter) => filter !== "stop"), + }) + } + /> + )} + {!filteringEnabled &&
{t("fullTextPolicy.stopFilterDisabled")}
} + + { + if (data.optionValue === undefined) { + return; + } + const next = { ...spec, language }; + if (data.optionValue === "") { + delete next.stopWordListKind; + delete next.addStopWords; + delete next.removeStopWords; + } else { + next.stopWordListKind = data.optionValue; + } + onChange(next); + }} + > + + {getStopwordPresets(packageName, language).map((preset) => ( + + ))} + + +
+ +