Add Mongo Proxy to Data Explorer (Standalone and Portal) (#1738)

* Mongo Proxy backend API

* merge main into current

* allow mongo proxy endpoints to be constants

* allow mongo proxy endpoints to be constants

* fix test

* check for allowed mongo proxy endpoint

* check for allowed mongo proxy endpoint

---------

Co-authored-by: Asier Isayas <aisayas@microsoft.com>
This commit is contained in:
Asier Isayas 2024-02-09 10:58:10 -05:00 committed by GitHub
parent f403b086ad
commit 8b0b3b07d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 376 additions and 18 deletions

View File

@ -145,4 +145,5 @@ src/Explorer/Notebook/temp/inputs/connected-editors/codemirror.tsx
src/Explorer/Tree/ResourceTreeAdapter.tsx
__mocks__/monaco-editor.ts
src/Explorer/Tree/ResourceTree.tsx
src/Utils/EndpointUtils.ts
src/Utils/PriorityBasedExecutionUtils.ts

View File

@ -211,6 +211,10 @@ export class HttpHeaders {
public static migrateOfferToAutopilot: string = "x-ms-cosmos-migrate-offer-to-autopilot";
}
export class ContentType {
public static applicationJson: string = "application/json";
}
export class ApiType {
// Mapped to hexadecimal values in the backend
public static readonly MongoDB: number = 1;

View File

@ -1,6 +1,6 @@
import { Constants as CosmosSDKConstants } from "@azure/cosmos";
import { MongoProxyEndpoints, allowedMongoProxyEndpoints_ToBeDeprecated, validateEndpoint } from "Utils/EndpointUtils";
import queryString from "querystring";
import { allowedMongoProxyEndpoints, validateEndpoint } from "Utils/EndpointValidation";
import { AuthType } from "../AuthType";
import { configContext } from "../ConfigContext";
import * as DataModels from "../Contracts/DataModels";
@ -10,7 +10,7 @@ import DocumentId from "../Explorer/Tree/DocumentId";
import { hasFlag } from "../Platform/Hosted/extractFeatures";
import { userContext } from "../UserContext";
import { logConsoleError } from "../Utils/NotificationConsoleUtils";
import { ApiType, HttpHeaders, HttpStatusCodes } from "./Constants";
import { ApiType, ContentType, HttpHeaders, HttpStatusCodes } from "./Constants";
import { MinimalQueryIterator } from "./IteratorUtilities";
import { sendMessage } from "./MessageHandler";
@ -62,6 +62,73 @@ export function queryDocuments(
isResourceList: boolean,
query: string,
continuationToken?: string,
): Promise<QueryResponse> {
if (!useMongoProxyEndpoint("resourcelist")) {
return queryDocuments_ToBeDeprecated(databaseId, collection, isResourceList, query, continuationToken);
}
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
const params = {
databaseID: databaseId,
collectionID: collection.id(),
resourceUrl: `${resourceEndpoint}dbs/${databaseId}/colls/${collection.id()}/docs/`,
resourceID: collection.rid,
resourceType: "docs",
subscriptionID: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
databaseAccountName: databaseAccount.name,
partitionKey:
collection && collection.partitionKey && !collection.partitionKey.systemKey
? collection.partitionKeyProperties?.[0]
: "",
query,
};
const endpoint = getFeatureEndpointOrDefault("resourcelist") || "";
const headers = {
...defaultHeaders,
...authHeaders(),
[CosmosSDKConstants.HttpHeaders.IsQuery]: "true",
[CosmosSDKConstants.HttpHeaders.PopulateQueryMetrics]: "true",
[CosmosSDKConstants.HttpHeaders.EnableScanInQuery]: "true",
[CosmosSDKConstants.HttpHeaders.EnableCrossPartitionQuery]: "true",
[CosmosSDKConstants.HttpHeaders.ParallelizeCrossPartitionQuery]: "true",
[HttpHeaders.contentType]: "application/query+json",
};
if (continuationToken) {
headers[CosmosSDKConstants.HttpHeaders.Continuation] = continuationToken;
}
const path = isResourceList ? "/resourcelist" : "";
return window
.fetch(`${endpoint}${path}`, {
method: "POST",
body: JSON.stringify(params),
headers,
})
.then(async (response) => {
if (response.ok) {
return {
continuationToken: response.headers.get(CosmosSDKConstants.HttpHeaders.Continuation),
documents: (await response.json()).Documents as DataModels.DocumentId[],
headers: response.headers,
};
}
await errorHandling(response, "querying documents", params);
return undefined;
});
}
function queryDocuments_ToBeDeprecated(
databaseId: string,
collection: Collection,
isResourceList: boolean,
query: string,
continuationToken?: string,
): Promise<QueryResponse> {
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
@ -122,6 +189,54 @@ export function readDocument(
databaseId: string,
collection: Collection,
documentId: DocumentId,
): Promise<DataModels.DocumentId> {
if (!useMongoProxyEndpoint("readDocument")) {
return readDocument_ToBeDeprecated(databaseId, collection, documentId);
}
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
const idComponents = documentId.self.split("/");
const path = idComponents.slice(0, 4).join("/");
const rid = encodeURIComponent(idComponents[5]);
const params = {
databaseID: databaseId,
collectionID: collection.id(),
resourceUrl: `${resourceEndpoint}${path}/${rid}`,
resourceID: rid,
resourceType: "docs",
subscriptionID: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
databaseAccountName: databaseAccount.name,
partitionKey:
documentId && documentId.partitionKey && !documentId.partitionKey.systemKey
? documentId.partitionKeyProperties?.[0]
: "",
};
const endpoint = getFeatureEndpointOrDefault("readDocument");
return window
.fetch(endpoint, {
method: "POST",
body: JSON.stringify(params),
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: ContentType.applicationJson,
},
})
.then(async (response) => {
if (response.ok) {
return response.json();
}
return await errorHandling(response, "reading document", params);
});
}
export function readDocument_ToBeDeprecated(
databaseId: string,
collection: Collection,
documentId: DocumentId,
): Promise<DataModels.DocumentId> {
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
@ -169,6 +284,51 @@ export function createDocument(
collection: Collection,
partitionKeyProperty: string,
documentContent: unknown,
): Promise<DataModels.DocumentId> {
if (!useMongoProxyEndpoint("createDocument")) {
return createDocument_ToBeDeprecated(databaseId, collection, partitionKeyProperty, documentContent);
}
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
const params = {
databaseID: databaseId,
collectionID: collection.id(),
resourceUrl: `${resourceEndpoint}dbs/${databaseId}/colls/${collection.id()}/docs/`,
resourceID: collection.rid,
resourceType: "docs",
subscriptionID: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
databaseAccountName: databaseAccount.name,
partitionKey:
collection && collection.partitionKey && !collection.partitionKey.systemKey ? partitionKeyProperty : "",
documentContent: JSON.stringify(documentContent),
};
const endpoint = getFeatureEndpointOrDefault("createDocument");
return window
.fetch(`${endpoint}/createDocument`, {
method: "POST",
body: JSON.stringify(params),
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: ContentType.applicationJson,
},
})
.then(async (response) => {
if (response.ok) {
return response.json();
}
return await errorHandling(response, "creating document", params);
});
}
export function createDocument_ToBeDeprecated(
databaseId: string,
collection: Collection,
partitionKeyProperty: string,
documentContent: unknown,
): Promise<DataModels.DocumentId> {
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
@ -208,6 +368,56 @@ export function updateDocument(
collection: Collection,
documentId: DocumentId,
documentContent: string,
): Promise<DataModels.DocumentId> {
if (!useMongoProxyEndpoint("updateDocument")) {
return updateDocument_ToBeDeprecated(databaseId, collection, documentId, documentContent);
}
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
const idComponents = documentId.self.split("/");
const path = idComponents.slice(0, 5).join("/");
const rid = encodeURIComponent(idComponents[5]);
const params = {
databaseID: databaseId,
collectionID: collection.id(),
resourceUrl: `${resourceEndpoint}${path}/${rid}`,
resourceID: rid,
resourceType: "docs",
subscriptionID: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
databaseAccountName: databaseAccount.name,
partitionKey:
documentId && documentId.partitionKey && !documentId.partitionKey.systemKey
? documentId.partitionKeyProperties?.[0]
: "",
documentContent,
};
const endpoint = getFeatureEndpointOrDefault("updateDocument");
return window
.fetch(endpoint, {
method: "PUT",
body: JSON.stringify(params),
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: ContentType.applicationJson,
[CosmosSDKConstants.HttpHeaders.PartitionKey]: JSON.stringify(documentId.partitionKeyHeader()),
},
})
.then(async (response) => {
if (response.ok) {
return response.json();
}
return await errorHandling(response, "updating document", params);
});
}
export function updateDocument_ToBeDeprecated(
databaseId: string,
collection: Collection,
documentId: DocumentId,
documentContent: string,
): Promise<DataModels.DocumentId> {
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
@ -237,7 +447,7 @@ export function updateDocument(
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: "application/json",
[HttpHeaders.contentType]: ContentType.applicationJson,
[CosmosSDKConstants.HttpHeaders.PartitionKey]: JSON.stringify(documentId.partitionKeyHeader()),
},
})
@ -250,6 +460,53 @@ export function updateDocument(
}
export function deleteDocument(databaseId: string, collection: Collection, documentId: DocumentId): Promise<void> {
if (!useMongoProxyEndpoint("deleteDocument")) {
deleteDocument_ToBeDeprecated(databaseId, collection, documentId);
}
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
const idComponents = documentId.self.split("/");
const path = idComponents.slice(0, 5).join("/");
const rid = encodeURIComponent(idComponents[5]);
const params = {
databaseID: databaseId,
collectionID: collection.id(),
resourceUrl: `${resourceEndpoint}${path}/${rid}`,
resourceID: rid,
resourceType: "docs",
subscriptionID: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
databaseAccountName: databaseAccount.name,
partitionKey:
documentId && documentId.partitionKey && !documentId.partitionKey.systemKey
? documentId.partitionKeyProperties?.[0]
: "",
};
const endpoint = getFeatureEndpointOrDefault("deleteDocument");
return window
.fetch(endpoint, {
method: "DELETE",
body: JSON.stringify(params),
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: ContentType.applicationJson,
},
})
.then(async (response) => {
if (response.ok) {
return undefined;
}
return await errorHandling(response, "deleting document", params);
});
}
export function deleteDocument_ToBeDeprecated(
databaseId: string,
collection: Collection,
documentId: DocumentId,
): Promise<void> {
const { databaseAccount } = userContext;
const resourceEndpoint = databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint;
const idComponents = documentId.self.split("/");
@ -277,7 +534,7 @@ export function deleteDocument(databaseId: string, collection: Collection, docum
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: "application/json",
[HttpHeaders.contentType]: ContentType.applicationJson,
[CosmosSDKConstants.HttpHeaders.PartitionKey]: JSON.stringify(documentId.partitionKeyHeader()),
},
})
@ -291,6 +548,52 @@ export function deleteDocument(databaseId: string, collection: Collection, docum
export function createMongoCollectionWithProxy(
params: DataModels.CreateCollectionParams,
): Promise<DataModels.Collection> {
if (!useMongoProxyEndpoint("createCollectionWithProxy")) {
createMongoCollectionWithProxy_ToBeDeprecated(params);
}
const { databaseAccount } = userContext;
const shardKey: string = params.partitionKey?.paths[0];
const createCollectionParams = {
databaseID: params.databaseId,
collectionID: params.collectionId,
resourceUrl: databaseAccount.properties.mongoEndpoint || databaseAccount.properties.documentEndpoint,
resourceID: "",
resourceType: "colls",
subscriptionID: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
databaseAccountName: databaseAccount.name,
partitionKey: shardKey,
isAutoscale: !!params.autoPilotMaxThroughput,
hasSharedThroughput: params.databaseLevelThroughput,
offerThroughput: params.autoPilotMaxThroughput || params.offerThroughput,
createDatabase: params.createNewDatabase,
isSharded: !!shardKey,
};
const endpoint = getFeatureEndpointOrDefault("createCollectionWithProxy");
return window
.fetch(`${endpoint}/createCollection`, {
method: "POST",
body: JSON.stringify(createCollectionParams),
headers: {
...defaultHeaders,
...authHeaders(),
[HttpHeaders.contentType]: ContentType.applicationJson,
},
})
.then(async (response) => {
if (response.ok) {
return response.json();
}
return await errorHandling(response, "creating collection", createCollectionParams);
});
}
export function createMongoCollectionWithProxy_ToBeDeprecated(
params: DataModels.CreateCollectionParams,
): Promise<DataModels.Collection> {
const { databaseAccount } = userContext;
const shardKey: string = params.partitionKey?.paths[0];
@ -334,13 +637,17 @@ export function createMongoCollectionWithProxy(
return await errorHandling(response, "creating collection", mongoParams);
});
}
export function getFeatureEndpointOrDefault(feature: string): string {
const endpoint =
let endpoint;
if (useMongoProxyEndpoint(feature)) {
endpoint = configContext.MONGO_PROXY_ENDPOINT;
} else {
endpoint =
hasFlag(userContext.features.mongoProxyAPIs, feature) &&
validateEndpoint(userContext.features.mongoProxyEndpoint, allowedMongoProxyEndpoints)
validateEndpoint(userContext.features.mongoProxyEndpoint, allowedMongoProxyEndpoints_ToBeDeprecated)
? userContext.features.mongoProxyEndpoint
: configContext.MONGO_BACKEND_ENDPOINT || configContext.BACKEND_ENDPOINT;
}
return getEndpoint(endpoint);
}
@ -349,8 +656,12 @@ export function getEndpoint(endpoint: string): string {
let url = endpoint + "/api/mongo/explorer";
if (userContext.authType === AuthType.EncryptedToken) {
if (endpoint === configContext.MONGO_PROXY_ENDPOINT) {
url = url.replace("api/mongo", "api/connectionstring/mongo");
} else {
url = url.replace("api/mongo", "api/guest/mongo");
}
}
return url;
}
@ -370,3 +681,10 @@ async function errorHandling(response: Response, action: string, params: unknown
export function getARMCreateCollectionEndpoint(params: DataModels.MongoParameters): string {
return `subscriptions/${params.sid}/resourceGroups/${params.rg}/providers/Microsoft.DocumentDB/databaseAccounts/${userContext.databaseAccount.name}/mongodbDatabases/${params.db}/collections/${params.coll}`;
}
function useMongoProxyEndpoint(api: string): boolean {
return (
configContext.NEW_MONGO_APIS?.includes(api) &&
[MongoProxyEndpoints.Development, MongoProxyEndpoints.MPAC].includes(configContext.MONGO_PROXY_ENDPOINT)
);
}

View File

@ -7,11 +7,12 @@ import {
allowedHostedExplorerEndpoints,
allowedJunoOrigins,
allowedMongoBackendEndpoints,
allowedMongoProxyEndpoints,
allowedMsalRedirectEndpoints,
defaultAllowedArmEndpoints,
defaultAllowedBackendEndpoints,
validateEndpoint,
} from "Utils/EndpointValidation";
} from "Utils/EndpointUtils";
export enum Platform {
Portal = "Portal",
@ -38,6 +39,8 @@ export interface ConfigContext {
ARCADIA_LIVY_ENDPOINT_DNS_ZONE: string;
BACKEND_ENDPOINT?: string;
MONGO_BACKEND_ENDPOINT?: string;
MONGO_PROXY_ENDPOINT?: string;
NEW_MONGO_APIS?: string[];
PROXY_PATH?: string;
JUNO_ENDPOINT: string;
GITHUB_CLIENT_ID: string;
@ -82,6 +85,15 @@ let configContext: Readonly<ConfigContext> = {
GITHUB_TEST_ENV_CLIENT_ID: "b63fc8cbf87fd3c6e2eb", // Registered OAuth app: https://github.com/organizations/AzureCosmosDBNotebooks/settings/applications/1777772
JUNO_ENDPOINT: JunoEndpoints.Prod,
BACKEND_ENDPOINT: "https://main.documentdb.ext.azure.com",
MONGO_PROXY_ENDPOINT: "https://cdb-ms-prod-mp.cosmos.azure.com",
NEW_MONGO_APIS: [
// "resourcelist",
// "createDocument",
// "readDocument",
// "updateDocument",
// "deleteDocument",
// "createCollectionWithProxy",
],
isTerminalEnabled: false,
isPhoenixEnabled: false,
};
@ -127,6 +139,10 @@ export function updateConfigContext(newContext: Partial<ConfigContext>): void {
delete newContext.BACKEND_ENDPOINT;
}
if (!validateEndpoint(newContext.MONGO_PROXY_ENDPOINT, allowedMongoProxyEndpoints)) {
delete newContext.MONGO_PROXY_ENDPOINT;
}
if (!validateEndpoint(newContext.MONGO_BACKEND_ENDPOINT, allowedMongoBackendEndpoints)) {
delete newContext.MONGO_BACKEND_ENDPOINT;
}

View File

@ -386,6 +386,7 @@ export interface DataExplorerInputsFrame {
dnsSuffix?: string;
serverId?: string;
extensionEndpoint?: string;
mongoProxyEndpoint?: string;
subscriptionType?: SubscriptionType;
quotaId?: string;
isTryCosmosDBSubscription?: boolean;

View File

@ -7,7 +7,7 @@ import { getCopilotEnabled, isCopilotFeatureRegistered } from "Explorer/QueryCop
import { IGalleryItem } from "Juno/JunoClient";
import { scheduleRefreshDatabaseResourceToken } from "Platform/Fabric/FabricUtil";
import { LocalStorageUtility, StorageKey } from "Shared/StorageUtility";
import { allowedNotebookServerUrls, validateEndpoint } from "Utils/EndpointValidation";
import { allowedNotebookServerUrls, validateEndpoint } from "Utils/EndpointUtils";
import { useQueryCopilot } from "hooks/useQueryCopilot";
import * as ko from "knockout";
import React from "react";

View File

@ -1,6 +1,6 @@
import ko from "knockout";
import { allowedJunoOrigins, validateEndpoint } from "Utils/EndpointValidation";
import { allowedJunoOrigins, validateEndpoint } from "Utils/EndpointUtils";
import { GetGithubClientId } from "Utils/GitHubUtils";
import ko from "knockout";
import { HttpHeaders, HttpStatusCodes } from "../Common/Constants";
import { configContext } from "../ConfigContext";
import * as DataModels from "../Contracts/DataModels";

View File

@ -2,7 +2,7 @@ import { configContext } from "ConfigContext";
import { useDialog } from "Explorer/Controls/Dialog";
import { Action } from "Shared/Telemetry/TelemetryConstants";
import { userContext } from "UserContext";
import { allowedJunoOrigins, validateEndpoint } from "Utils/EndpointValidation";
import { allowedJunoOrigins, validateEndpoint } from "Utils/EndpointUtils";
import { useQueryCopilot } from "hooks/useQueryCopilot";
import promiseRetry, { AbortError } from "p-retry";
import {

View File

@ -67,7 +67,23 @@ export const PortalBackendIPs: { [key: string]: string[] } = {
//usnat: ["7.28.202.68"],
};
export class MongoProxyEndpoints {
public static readonly Development: string = "https://localhost:7238";
public static readonly MPAC: string = "https://cdb-ms-mpac-mp.cosmos.azure.com";
public static readonly Prod: string = "https://cdb-ms-prod-mp.cosmos.azure.com";
public static readonly Fairfax: string = "https://cdb-ff-prod-mp.cosmos.azure.us";
public static readonly Mooncake: string = "https://cdb-mc-prod-mp.cosmos.azure.cn";
}
export const allowedMongoProxyEndpoints: ReadonlyArray<string> = [
MongoProxyEndpoints.Development,
MongoProxyEndpoints.MPAC,
MongoProxyEndpoints.Prod,
MongoProxyEndpoints.Fairfax,
MongoProxyEndpoints.Mooncake,
];
export const allowedMongoProxyEndpoints_ToBeDeprecated: ReadonlyArray<string> = [
"https://main.documentdb.ext.azure.com",
"https://main.documentdb.ext.azure.cn",
"https://main.documentdb.ext.azure.us",

View File

@ -1,7 +1,7 @@
import { resetConfigContext, updateConfigContext } from "ConfigContext";
import { DatabaseAccount, IpRule } from "Contracts/DataModels";
import { updateUserContext } from "UserContext";
import { PortalBackendIPs } from "Utils/EndpointValidation";
import { PortalBackendIPs } from "Utils/EndpointUtils";
import { getNetworkSettingsWarningMessage } from "./NetworkUtility";
describe("NetworkUtility tests", () => {

View File

@ -1,7 +1,7 @@
import { configContext } from "ConfigContext";
import { checkFirewallRules } from "Explorer/Tabs/Shared/CheckFirewallRules";
import { userContext } from "UserContext";
import { PortalBackendIPs } from "Utils/EndpointValidation";
import { PortalBackendIPs } from "Utils/EndpointUtils";
export const getNetworkSettingsWarningMessage = async (
setStateFunc: (warningMessage: string) => void,

View File

@ -478,6 +478,7 @@ function updateContextsFromPortalMessage(inputs: DataExplorerInputsFrame) {
updateConfigContext({
BACKEND_ENDPOINT: inputs.extensionEndpoint || configContext.BACKEND_ENDPOINT,
ARM_ENDPOINT: normalizeArmEndpoint(inputs.csmEndpoint || configContext.ARM_ENDPOINT),
MONGO_PROXY_ENDPOINT: inputs.mongoProxyEndpoint,
});
updateUserContext({

View File

@ -112,6 +112,7 @@
"./src/Utils/BlobUtils.ts",
"./src/Utils/CapabilityUtils.ts",
"./src/Utils/CloudUtils.ts",
"./src/Utils/EndpointUtils.ts",
"./src/Utils/GitHubUtils.test.ts",
"./src/Utils/GitHubUtils.ts",
"./src/Utils/MessageValidation.test.ts",