copy job process performance enhancement (#2273)

This commit is contained in:
BChoudhury-ms
2025-12-05 11:49:25 +05:30
committed by GitHub
parent d060f22357
commit fa18b85364
16 changed files with 159 additions and 138 deletions

View File

@@ -44,8 +44,8 @@ export const getDatabaseEndpoint = (apiType: ApiType): string => {
return "gremlinDatabases";
case "Tables":
return "tables";
default:
case "SQL":
default:
return "sqlDatabases";
}
};
@@ -58,8 +58,8 @@ export const getCollectionEndpoint = (apiType: ApiType): string => {
return "tables";
case "Gremlin":
return "graphs";
default:
case "SQL":
default:
return "containers";
}
};

View File

@@ -36,14 +36,6 @@ export interface DatabaseAccountSystemData {
export interface DatabaseAccountBackupPolicy {
type: string;
/* periodicModeProperties?: {
backupIntervalInMinutes: number;
backupRetentionIntervalInHours: number;
backupStorageRedundancy: string;
};
continuousModeProperties?: {
tier: string;
}; */
}
export interface DatabaseAccountExtendedProperties {
@@ -73,6 +65,7 @@ export interface DatabaseAccountExtendedProperties {
publicNetworkAccess?: string;
enablePriorityBasedExecution?: boolean;
vcoreMongoEndpoint?: string;
enableAllVersionsAndDeletesChangeFeed?: boolean;
}
export interface DatabaseAccountResponseLocation {

View File

@@ -23,6 +23,7 @@ import {
extractErrorMessage,
formatUTCDateTime,
getAccountDetailsFromResourceId,
isIntraAccountCopy,
} from "../CopyJobUtils";
import CreateCopyJobScreensProvider from "../CreateCopyJob/Screens/CreateCopyJobScreensProvider";
import { CopyJobActions, CopyJobStatusType } from "../Enums/CopyJobEnums";
@@ -75,7 +76,6 @@ export const getCopyJobs = async (): Promise<CopyJobType[]> => {
}
copyJobsAbortController = null;
/* added a lower bound to "0" and upper bound to "100" */
const calculateCompletionPercentage = (processed: number, total: number): number => {
if (
typeof processed !== "number" ||
@@ -139,11 +139,12 @@ export const submitCreateCopyJob = async (state: CopyJobContextState, onSuccess:
const { subscriptionId, resourceGroup, accountName } = getAccountDetailsFromResourceId(
userContext.databaseAccount?.id || "",
);
const isSameAccount = isIntraAccountCopy(source?.account?.id, target?.account?.id);
const body = {
properties: {
source: {
component: "CosmosDBSql",
remoteAccountName: source?.account?.name,
...(isSameAccount ? {} : { remoteAccountName: source?.account?.name }),
databaseName: source?.databaseId,
containerName: source?.containerId,
},

View File

@@ -55,13 +55,15 @@ export default {
"To copy data from the source to the destination container, ensure that the managed identity of the destination account has read access to the source account by completing the following steps.",
intraAccountOnlineDescription: (accountName: string) =>
`Follow the steps below to enable online copy on your "${accountName}" account.`,
commonConfiguration: {
title: "Common configuration",
description: "Basic permissions required for copy operations",
crossAccountConfiguration: {
title: "Cross-account container copy",
description: (sourceAccount: string, destinationAccount: string) =>
`Please follow the instruction below to grant requisite permissions to copy data from "${sourceAccount}" to "${destinationAccount}".`,
},
onlineConfiguration: {
title: "Online copy configuration",
description: "Additional permissions required for online copy operations",
title: "Online container copy",
description: (accountName: string) =>
`Please follow the instructions below to enable online copy on your "${accountName}" account.`,
},
},
toggleBtn: {
@@ -129,10 +131,17 @@ export default {
},
onlineCopyEnabled: {
title: "Online copy enabled",
description: (accountName: string) => `Enable Online copy on "${accountName}".`,
description: (accountName: string) =>
`Enable online container copy by clicking the button below on your "${accountName}" account.`,
hrefText: "Learn more about online copy jobs",
href: "https://learn.microsoft.com/en-us/azure/cosmos-db/container-copy?tabs=online-copy&pivots=api-nosql#enable-online-copy",
buttonText: "Enable Online Copy",
validateAllVersionsAndDeletesChangeFeedSpinnerLabel:
"Validating All versions and deletes change feed mode (preview)...",
enablingAllVersionsAndDeletesChangeFeedSpinnerLabel:
"Enabling All versions and deletes change feed mode (preview)...",
enablingOnlineCopySpinnerLabel: (accountName: string) =>
`Enabling online copy on your "${accountName}" account ...`,
},
MonitorJobs: {
Columns: {

View File

@@ -1,5 +1,5 @@
import { Link, Stack, Text, Toggle } from "@fluentui/react";
import React, { useCallback } from "react";
import React from "react";
import { logError } from "../../../../../Common/Logger";
import { assignRole } from "../../../../../Utils/arm/RbacUtils";
import ContainerCopyMessages from "../../../ContainerCopyMessages";
@@ -25,7 +25,7 @@ const AddReadPermissionToDefaultIdentity: React.FC<AddReadPermissionToDefaultIde
const { copyJobState, setCopyJobState, setContextError } = useCopyJobContext();
const [readPermissionAssigned, onToggle] = useToggle(false);
const handleAddReadPermission = useCallback(async () => {
const handleAddReadPermission = async () => {
const { source, target } = copyJobState;
const selectedSourceAccount = source?.account;
try {
@@ -56,7 +56,7 @@ const AddReadPermissionToDefaultIdentity: React.FC<AddReadPermissionToDefaultIde
} finally {
setLoading(false);
}
}, [copyJobState, setCopyJobState, setContextError]);
};
return (
<Stack className="defaultManagedIdentityContainer" tokens={{ childrenGap: 15, padding: "0 0 0 20px" }}>

View File

@@ -20,6 +20,7 @@ const validatorFn: AccountValidatorFn = (prev: DatabaseAccount, next: DatabaseAc
const OnlineCopyEnabled: React.FC = () => {
const [loading, setLoading] = React.useState(false);
const [loaderMessage, setLoaderMessage] = React.useState("");
const [showRefreshButton, setShowRefreshButton] = React.useState(false);
const intervalRef = React.useRef<NodeJS.Timeout | null>(null);
const timeoutRef = React.useRef<NodeJS.Timeout | null>(null);
@@ -75,6 +76,21 @@ const OnlineCopyEnabled: React.FC = () => {
setShowRefreshButton(false);
try {
setLoaderMessage(ContainerCopyMessages.onlineCopyEnabled.validateAllVersionsAndDeletesChangeFeedSpinnerLabel);
const sourAccountBeforeUpdate = await fetchDatabaseAccount(
sourceSubscriptionId,
sourceResourceGroup,
sourceAccountName,
);
if (!sourAccountBeforeUpdate?.properties.enableAllVersionsAndDeletesChangeFeed) {
setLoaderMessage(ContainerCopyMessages.onlineCopyEnabled.enablingAllVersionsAndDeletesChangeFeedSpinnerLabel);
await updateDatabaseAccount(sourceSubscriptionId, sourceResourceGroup, sourceAccountName, {
properties: {
enableAllVersionsAndDeletesChangeFeed: true,
},
});
}
setLoaderMessage(ContainerCopyMessages.onlineCopyEnabled.enablingOnlineCopySpinnerLabel(sourceAccountName));
await updateDatabaseAccount(sourceSubscriptionId, sourceResourceGroup, sourceAccountName, {
properties: {
enableAllVersionsAndDeletesChangeFeed: true,
@@ -120,7 +136,7 @@ const OnlineCopyEnabled: React.FC = () => {
return (
<Stack className="onlineCopyContainer" tokens={{ childrenGap: 15, padding: "0 0 0 20px" }}>
<LoadingOverlay isLoading={loading} label={ContainerCopyMessages.popoverOverlaySpinnerLabel} />
<LoadingOverlay isLoading={loading} label={loaderMessage} />
<Stack.Item className="info-message">
{ContainerCopyMessages.onlineCopyEnabled.description(source?.account?.name || "")}&ensp;
<Link href={ContainerCopyMessages.onlineCopyEnabled.href} target="_blank" rel="noopener noreferrer">

View File

@@ -44,10 +44,9 @@ const useManagedIdentity = (
const errorMessage = error.message || "Error enabling system-assigned managed identity. Please try again later.";
logError(errorMessage, "CopyJob/useManagedIdentity.handleAddSystemIdentity");
setContextError(errorMessage);
} finally {
setLoading(false);
}
}, [copyJobState, updateIdentityFn, setCopyJobState]);
}, [copyJobState?.target?.account?.id, updateIdentityFn, setCopyJobState]);
return { loading, handleAddSystemIdentity };
};

View File

@@ -186,15 +186,20 @@ const usePermissionSections = (state: CopyJobContextState): PermissionGroupConfi
const groupsToValidate = useMemo(() => {
const isSameAccount = isIntraAccountCopy(sourceAccount.accountId, targetAccount.accountId);
const commonSections = isSameAccount ? [] : [...PERMISSION_SECTIONS_CONFIG];
const crossAccountSections = isSameAccount ? [] : [...PERMISSION_SECTIONS_CONFIG];
const groups: PermissionGroupConfig[] = [];
const sourceAccountName = state.source?.account?.name || "";
const targetAccountName = state.target?.account?.name || "";
if (commonSections.length > 0) {
if (crossAccountSections.length > 0) {
groups.push({
id: "commonConfigs",
title: ContainerCopyMessages.assignPermissions.commonConfiguration.title,
description: ContainerCopyMessages.assignPermissions.commonConfiguration.description,
sections: commonSections,
id: "crossAccountConfigs",
title: ContainerCopyMessages.assignPermissions.crossAccountConfiguration.title,
description: ContainerCopyMessages.assignPermissions.crossAccountConfiguration.description(
sourceAccountName,
targetAccountName,
),
sections: crossAccountSections,
});
}
@@ -202,7 +207,7 @@ const usePermissionSections = (state: CopyJobContextState): PermissionGroupConfi
groups.push({
id: "onlineConfigs",
title: ContainerCopyMessages.assignPermissions.onlineConfiguration.title,
description: ContainerCopyMessages.assignPermissions.onlineConfiguration.description,
description: ContainerCopyMessages.assignPermissions.onlineConfiguration.description(sourceAccountName),
sections: [...PERMISSION_SECTIONS_FOR_ONLINE_JOBS],
});
}

View File

@@ -11,25 +11,19 @@ export function useDropdownOptions(
subscriptionOptions: DropdownOptionType[];
accountOptions: DropdownOptionType[];
} {
const subscriptionOptions = React.useMemo(
() =>
subscriptions?.map((sub) => ({
key: sub.subscriptionId,
text: sub.displayName,
data: sub,
})) || [],
[subscriptions],
);
const subscriptionOptions =
subscriptions?.map((sub) => ({
key: sub.subscriptionId,
text: sub.displayName,
data: sub,
})) || [];
const accountOptions = React.useMemo(
() =>
accounts?.map((account) => ({
key: account.id,
text: account.name,
data: account,
})) || [],
[accounts],
);
const accountOptions =
accounts?.map((account) => ({
key: account.id,
text: account.name,
data: account,
})) || [];
return { subscriptionOptions, accountOptions };
}
@@ -38,45 +32,42 @@ type setCopyJobStateType = CopyJobContextProviderType["setCopyJobState"];
export function useEventHandlers(setCopyJobState: setCopyJobStateType) {
const { setValidationCache } = useCopyJobPrerequisitesCache();
const handleSelectSourceAccount = React.useCallback(
(type: "subscription" | "account", data: (Subscription & DatabaseAccount) | undefined) => {
setCopyJobState((prevState: CopyJobContextState) => {
if (type === "subscription") {
return {
...prevState,
source: {
...prevState.source,
subscription: data || null,
account: null,
},
};
}
if (type === "account") {
return {
...prevState,
source: {
...prevState.source,
account: data || null,
},
};
}
return prevState;
});
setValidationCache(new Map<string, boolean>());
},
[setCopyJobState, setValidationCache],
);
const handleSelectSourceAccount = (
type: "subscription" | "account",
data: (Subscription & DatabaseAccount) | undefined,
) => {
setCopyJobState((prevState: CopyJobContextState) => {
if (type === "subscription") {
return {
...prevState,
source: {
...prevState.source,
subscription: data || null,
account: null,
},
};
}
if (type === "account") {
return {
...prevState,
source: {
...prevState.source,
account: data || null,
},
};
}
return prevState;
});
setValidationCache(new Map<string, boolean>());
};
const handleMigrationTypeChange = React.useCallback(
(_ev?: React.FormEvent<HTMLElement>, checked?: boolean) => {
setCopyJobState((prevState: CopyJobContextState) => ({
...prevState,
migrationType: checked ? CopyJobMigrationType.Offline : CopyJobMigrationType.Online,
}));
setValidationCache(new Map<string, boolean>());
},
[setCopyJobState, setValidationCache],
);
const handleMigrationTypeChange = React.useCallback((_ev?: React.FormEvent<HTMLElement>, checked?: boolean) => {
setCopyJobState((prevState: CopyJobContextState) => ({
...prevState,
migrationType: checked ? CopyJobMigrationType.Offline : CopyJobMigrationType.Online,
}));
setValidationCache(new Map<string, boolean>());
}, []);
return { handleSelectSourceAccount, handleMigrationTypeChange };
}

View File

@@ -7,7 +7,7 @@ import ContainerCopyMessages from "../../../ContainerCopyMessages";
import { useCopyJobContext } from "../../../Context/CopyJobContext";
import { DatabaseContainerSection } from "./components/DatabaseContainerSection";
import { dropDownChangeHandler } from "./Events/DropDownChangeHandler";
import { useMemoizedSourceAndTargetData } from "./memoizedData";
import { useSourceAndTargetData } from "./memoizedData";
type SelectSourceAndTargetContainers = {
showAddCollectionPanel?: () => void;
@@ -16,31 +16,35 @@ type SelectSourceAndTargetContainers = {
const SelectSourceAndTargetContainers = ({ showAddCollectionPanel }: SelectSourceAndTargetContainers) => {
const { copyJobState, setCopyJobState } = useCopyJobContext();
const { source, target, sourceDbParams, sourceContainerParams, targetDbParams, targetContainerParams } =
useMemoizedSourceAndTargetData(copyJobState);
useSourceAndTargetData(copyJobState);
const sourceDatabases = useDatabases(...sourceDbParams) || [];
const sourceContainers = useDataContainers(...sourceContainerParams) || [];
const targetDatabases = useDatabases(...targetDbParams) || [];
const targetContainers = useDataContainers(...targetContainerParams) || [];
if (!source) {
return null;
}
const sourceDatabases = useDatabases(...sourceDbParams);
const sourceContainers = useDataContainers(...sourceContainerParams);
const targetDatabases = useDatabases(...targetDbParams);
const targetContainers = useDataContainers(...targetContainerParams);
const sourceDatabaseOptions = React.useMemo(
() => sourceDatabases.map((db: DatabaseModel) => ({ key: db.name, text: db.name, data: db })),
() => sourceDatabases?.map((db: DatabaseModel) => ({ key: db.name, text: db.name, data: db })) || [],
[sourceDatabases],
);
const sourceContainerOptions = React.useMemo(
() => sourceContainers.map((c: DatabaseModel) => ({ key: c.name, text: c.name, data: c })),
() => sourceContainers?.map((c: DatabaseModel) => ({ key: c.name, text: c.name, data: c })) || [],
[sourceContainers],
);
const targetDatabaseOptions = React.useMemo(
() => targetDatabases.map((db: DatabaseModel) => ({ key: db.name, text: db.name, data: db })),
() => targetDatabases?.map((db: DatabaseModel) => ({ key: db.name, text: db.name, data: db })) || [],
[targetDatabases],
);
const targetContainerOptions = React.useMemo(
() => targetContainers.map((c: DatabaseModel) => ({ key: c.name, text: c.name, data: c })),
() => targetContainers?.map((c: DatabaseModel) => ({ key: c.name, text: c.name, data: c })) || [],
[targetContainers],
);
const onDropdownChange = React.useCallback(dropDownChangeHandler(setCopyJobState), [setCopyJobState]);
const onDropdownChange = dropDownChangeHandler(setCopyJobState);
return (
<Stack className="selectSourceAndTargetContainers" tokens={{ childrenGap: 25 }}>

View File

@@ -1,8 +1,7 @@
import React from "react";
import { getAccountDetailsFromResourceId } from "../../../CopyJobUtils";
import { CopyJobContextState, DatabaseParams, DataContainerParams } from "../../../Types/CopyJobTypes";
export function useMemoizedSourceAndTargetData(copyJobState: CopyJobContextState) {
export function useSourceAndTargetData(copyJobState: CopyJobContextState) {
const { source, target } = copyJobState ?? {};
const selectedSourceAccount = source?.account;
const selectedTargetAccount = target?.account;
@@ -17,27 +16,22 @@ export function useMemoizedSourceAndTargetData(copyJobState: CopyJobContextState
accountName: targetAccountName,
} = getAccountDetailsFromResourceId(selectedTargetAccount?.id);
const sourceDbParams = React.useMemo(
() => [sourceSubscriptionId, sourceResourceGroup, sourceAccountName, "SQL"] as DatabaseParams,
[sourceSubscriptionId, sourceResourceGroup, sourceAccountName],
);
const sourceContainerParams = React.useMemo(
() =>
[sourceSubscriptionId, sourceResourceGroup, sourceAccountName, source?.databaseId, "SQL"] as DataContainerParams,
[sourceSubscriptionId, sourceResourceGroup, sourceAccountName, source?.databaseId],
);
const targetDbParams = React.useMemo(
() => [targetSubscriptionId, targetResourceGroup, targetAccountName, "SQL"] as DatabaseParams,
[targetSubscriptionId, targetResourceGroup, targetAccountName],
);
const targetContainerParams = React.useMemo(
() =>
[targetSubscriptionId, targetResourceGroup, targetAccountName, target?.databaseId, "SQL"] as DataContainerParams,
[targetSubscriptionId, targetResourceGroup, targetAccountName, target?.databaseId],
);
const sourceDbParams = [sourceSubscriptionId, sourceResourceGroup, sourceAccountName, "SQL"] as DatabaseParams;
const sourceContainerParams = [
sourceSubscriptionId,
sourceResourceGroup,
sourceAccountName,
source?.databaseId,
"SQL",
] as DataContainerParams;
const targetDbParams = [targetSubscriptionId, targetResourceGroup, targetAccountName, "SQL"] as DatabaseParams;
const targetContainerParams = [
targetSubscriptionId,
targetResourceGroup,
targetAccountName,
target?.databaseId,
"SQL",
] as DataContainerParams;
return { source, target, sourceDbParams, sourceContainerParams, targetDbParams, targetContainerParams };
}

View File

@@ -1,4 +1,5 @@
import { FontIcon, getTheme, mergeStyles, mergeStyleSets, Spinner, SpinnerSize, Stack, Text } from "@fluentui/react";
import PropTypes from "prop-types";
import React from "react";
import ContainerCopyMessages from "../../ContainerCopyMessages";
import { CopyJobStatusType } from "../../Enums/CopyJobEnums";
@@ -34,7 +35,11 @@ const iconMap: Partial<Record<CopyJobStatusType, string>> = {
[CopyJobStatusType.Completed]: "CompletedSolid",
};
const CopyJobStatusWithIcon: React.FC<{ status: CopyJobStatusType }> = ({ status }) => {
export interface CopyJobStatusWithIconProps {
status: CopyJobStatusType;
}
const CopyJobStatusWithIcon: React.FC<CopyJobStatusWithIconProps> = React.memo(({ status }) => {
const statusText = ContainerCopyMessages.MonitorJobs.Status[status] || "Unknown";
const isSpinnerStatus = [
@@ -57,6 +62,11 @@ const CopyJobStatusWithIcon: React.FC<{ status: CopyJobStatusType }> = ({ status
<Text>{statusText}</Text>
</Stack>
);
});
CopyJobStatusWithIcon.displayName = "CopyJobStatusWithIcon";
CopyJobStatusWithIcon.propTypes = {
status: PropTypes.oneOf(Object.values(CopyJobStatusType)).isRequired,
};
export default CopyJobStatusWithIcon;

View File

@@ -25,4 +25,4 @@ const CopyJobsNotFound: React.FC<CopyJobsNotFoundProps> = ({ explorer }) => {
);
};
export default CopyJobsNotFound;
export default React.memo(CopyJobsNotFound);

View File

@@ -1,9 +1,11 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable react/prop-types */
import {
ConstrainMode,
DetailsListLayoutMode,
DetailsRow,
IColumn,
IDetailsRowProps,
ScrollablePane,
ScrollbarVisibility,
ShimmeredDetailsList,
@@ -58,22 +60,19 @@ const CopyJobsList: React.FC<CopyJobsListProps> = ({ jobs, handleActionClick, pa
setStartIndex(0);
};
const columns: IColumn[] = React.useMemo(
() => getColumns(handleSort, handleActionClick, sortedColumnKey, isSortedDescending),
[handleSort, handleActionClick, sortedColumnKey, isSortedDescending],
);
const columns: IColumn[] = getColumns(handleSort, handleActionClick, sortedColumnKey, isSortedDescending);
const _handleRowClick = React.useCallback((job: CopyJobType) => {
const _handleRowClick = (job: CopyJobType) => {
openCopyJobDetailsPanel(job);
}, []);
};
const _onRenderRow = React.useCallback((props: any) => {
const _onRenderRow = (props: IDetailsRowProps) => {
return (
<div onClick={_handleRowClick.bind(null, props.item)}>
<DetailsRow {...props} styles={{ root: { cursor: "pointer" } }} />
</div>
);
}, []);
};
return (
<div style={styles.container}>

View File

@@ -4,13 +4,14 @@ import ShimmerTree, { IndentLevel } from "Common/ShimmerTree/ShimmerTree";
import Explorer from "Explorer/Explorer";
import React, { forwardRef, useEffect, useImperativeHandle } from "react";
import { getCopyJobs, updateCopyJobStatus } from "../Actions/CopyJobActions";
import { convertToCamelCase } from "../CopyJobUtils";
import { convertToCamelCase, isEqual } from "../CopyJobUtils";
import { CopyJobStatusType } from "../Enums/CopyJobEnums";
import CopyJobsNotFound from "../MonitorCopyJobs/Components/CopyJobs.NotFound";
import { CopyJobType, JobActionUpdatorType } from "../Types/CopyJobTypes";
import CopyJobsList from "./Components/CopyJobsList";
const FETCH_INTERVAL_MS = 30 * 1000;
const SHIMMER_INDENT_LEVELS: IndentLevel[] = Array(7).fill({ level: 0, width: "100%" });
interface MonitorCopyJobsProps {
explorer: Explorer;
@@ -27,8 +28,6 @@ const MonitorCopyJobs = forwardRef<MonitorCopyJobsRef, MonitorCopyJobsProps>(({
const isUpdatingRef = React.useRef(false);
const isFirstFetchRef = React.useRef(true);
const indentLevels = React.useMemo<IndentLevel[]>(() => Array(7).fill({ level: 0, width: "100%" }), []);
const fetchJobs = React.useCallback(async () => {
if (isUpdatingRef.current) {
return;
@@ -41,8 +40,7 @@ const MonitorCopyJobs = forwardRef<MonitorCopyJobsRef, MonitorCopyJobsProps>(({
const response = await getCopyJobs();
setJobs((prevJobs) => {
const isSame = JSON.stringify(prevJobs) === JSON.stringify(response);
return isSame ? prevJobs : response;
return isEqual(prevJobs, response) ? prevJobs : response;
});
} catch (error) {
setError(error.message || "Failed to load copy jobs. Please try again later.");
@@ -111,7 +109,9 @@ const MonitorCopyJobs = forwardRef<MonitorCopyJobsRef, MonitorCopyJobsProps>(({
return (
<Stack className="monitorCopyJobs flexContainer">
{loading && <ShimmerTree indentLevels={indentLevels} style={{ width: "100%", padding: "1rem 2.5rem" }} />}
{loading && (
<ShimmerTree indentLevels={SHIMMER_INDENT_LEVELS} style={{ width: "100%", padding: "1rem 2.5rem" }} />
)}
{error && (
<MessageBar messageBarType={MessageBarType.error} isMultiline={false} onDismiss={() => setError(null)}>
{error}

View File

@@ -2,7 +2,7 @@ import { DatabaseAccount } from "Contracts/DataModels";
import { userContext } from "UserContext";
import { buildArmUrl } from "Utils/arm/armUtils";
const apiVersion = "2025-04-15";
const apiVersion = "2025-05-01-preview";
export type FetchAccountDetailsParams = {
subscriptionId: string;
resourceGroupName: string;