Cleanup Synapse+Spark Logic (#813)

This commit is contained in:
Steve Faulkner
2021-05-25 14:46:52 -05:00
committed by GitHub
parent 522fdc69ab
commit e7e15c54b3
22 changed files with 46 additions and 6746 deletions

View File

@@ -6,40 +6,34 @@ import _ from "underscore";
import { AuthType } from "../AuthType";
import { BindingHandlersRegisterer } from "../Bindings/BindingHandlersRegisterer";
import * as Constants from "../Common/Constants";
import { ExplorerMetrics, HttpStatusCodes } from "../Common/Constants";
import { ExplorerMetrics } from "../Common/Constants";
import { readCollection } from "../Common/dataAccess/readCollection";
import { readDatabases } from "../Common/dataAccess/readDatabases";
import { getErrorMessage, getErrorStack, handleError } from "../Common/ErrorHandlingUtils";
import * as Logger from "../Common/Logger";
import { sendCachedDataMessage } from "../Common/MessageHandler";
import { QueriesClient } from "../Common/QueriesClient";
import { Splitter, SplitterBounds, SplitterDirection } from "../Common/Splitter";
import { configContext, Platform } from "../ConfigContext";
import * as DataModels from "../Contracts/DataModels";
import { MessageTypes } from "../Contracts/ExplorerContracts";
import * as ViewModels from "../Contracts/ViewModels";
import { GitHubClient } from "../GitHub/GitHubClient";
import { GitHubOAuthService } from "../GitHub/GitHubOAuthService";
import { IGalleryItem, JunoClient } from "../Juno/JunoClient";
import { NotebookWorkspaceManager } from "../NotebookWorkspaceManager/NotebookWorkspaceManager";
import { ResourceProviderClientFactory } from "../ResourceProvider/ResourceProviderClientFactory";
import { RouteHandler } from "../RouteHandlers/RouteHandler";
import { trackEvent } from "../Shared/appInsights";
import * as SharedConstants from "../Shared/Constants";
import { ExplorerSettings } from "../Shared/ExplorerSettings";
import { Action, ActionModifiers } from "../Shared/Telemetry/TelemetryConstants";
import * as TelemetryProcessor from "../Shared/Telemetry/TelemetryProcessor";
import { ArcadiaResourceManager } from "../SparkClusterManager/ArcadiaResourceManager";
import { updateUserContext, userContext } from "../UserContext";
import { userContext } from "../UserContext";
import { getCollectionName, getDatabaseName, getUploadName } from "../Utils/APITypeUtils";
import { decryptJWTToken, getAuthorizationHeader } from "../Utils/AuthorizationUtils";
import { update } from "../Utils/arm/generatedClients/cosmos/databaseAccounts";
import { getAuthorizationHeader } from "../Utils/AuthorizationUtils";
import { stringToBlob } from "../Utils/BlobUtils";
import { isCapabilityEnabled } from "../Utils/CapabilityUtils";
import { fromContentUri, toRawContentUri } from "../Utils/GitHubUtils";
import * as NotificationConsoleUtils from "../Utils/NotificationConsoleUtils";
import { logConsoleError, logConsoleInfo, logConsoleProgress } from "../Utils/NotificationConsoleUtils";
import * as ComponentRegisterer from "./ComponentRegisterer";
import { ArcadiaWorkspaceItem } from "./Controls/Arcadia/ArcadiaMenuPicker";
import { CommandButtonComponentProps } from "./Controls/CommandButton/CommandButtonComponent";
import { DialogProps, TextFieldProps, useDialog } from "./Controls/Dialog";
import { GalleryTab as GalleryTabKind } from "./Controls/NotebookGallery/GalleryViewerComponent";
@@ -55,7 +49,6 @@ import { AddCollectionPanel } from "./Panes/AddCollectionPanel";
import { AddDatabasePanel } from "./Panes/AddDatabasePanel/AddDatabasePanel";
import { BrowseQueriesPane } from "./Panes/BrowseQueriesPane/BrowseQueriesPane";
import { CassandraAddCollectionPane } from "./Panes/CassandraAddCollectionPane/CassandraAddCollectionPane";
import { ContextualPaneBase } from "./Panes/ContextualPaneBase";
import { DeleteCollectionConfirmationPane } from "./Panes/DeleteCollectionConfirmationPane/DeleteCollectionConfirmationPane";
import { DeleteDatabaseConfirmationPanel } from "./Panes/DeleteDatabaseConfirmationPanel";
import { ExecuteSprocParamsPane } from "./Panes/ExecuteSprocParamsPane/ExecuteSprocParamsPane";
@@ -104,7 +97,6 @@ export default class Explorer {
public isServerlessEnabled: ko.Computed<boolean>;
public isAccountReady: ko.Observable<boolean>;
public canSaveQueries: ko.Computed<boolean>;
public features: ko.Observable<any>;
public queriesClient: QueriesClient;
public tableDataClient: TableDataClient;
public splitter: Splitter;
@@ -115,7 +107,6 @@ export default class Explorer {
private setInProgressConsoleDataIdToBeDeleted: (id: string) => void;
// Panes
public contextPanes: ContextualPaneBase[];
public openSidePanel: (headerText: string, panelContent: JSX.Element, onClose?: () => void) => void;
public closeSidePanel: () => void;
@@ -139,16 +130,10 @@ export default class Explorer {
public isTabsContentExpanded: ko.Observable<boolean>;
public tabsManager: TabsManager;
// Contextual panes
private gitHubClient: GitHubClient;
public gitHubOAuthService: GitHubOAuthService;
public junoClient: JunoClient;
// features
public isPublishNotebookPaneEnabled: ko.Observable<boolean>;
public isHostedDataExplorerEnabled: ko.Computed<boolean>;
public isMongoIndexingEnabled: ko.Observable<boolean>;
public canExceedMaximumValue: ko.Computed<boolean>;
public isSchemaEnabled: ko.Computed<boolean>;
// Notebooks
@@ -157,11 +142,6 @@ export default class Explorer {
public notebookServerInfo: ko.Observable<DataModels.NotebookWorkspaceConnectionInfo>;
public notebookWorkspaceManager: NotebookWorkspaceManager;
public sparkClusterConnectionInfo: ko.Observable<DataModels.SparkClusterConnectionInfo>;
public isSparkEnabled: ko.Observable<boolean>;
public isSparkEnabledForAccount: ko.Observable<boolean>;
public arcadiaToken: ko.Observable<string>;
public arcadiaWorkspaces: ko.ObservableArray<ArcadiaWorkspaceItem>;
public hasStorageAnalyticsAfecFeature: ko.Observable<boolean>;
public isSynapseLinkUpdating: ko.Observable<boolean>;
public memoryUsageInfo: ko.Observable<DataModels.MemoryUsageInfo>;
public notebookManager?: NotebookManager;
@@ -170,7 +150,6 @@ export default class Explorer {
private _isInitializingNotebooks: boolean;
private notebookBasePath: ko.Observable<string>;
private _arcadiaManager: ArcadiaResourceManager;
private notebookToImport: {
name: string;
content: string;
@@ -182,8 +161,6 @@ export default class Explorer {
private static readonly MaxNbDatabasesToAutoExpand = 5;
constructor(params?: ExplorerParams) {
this.gitHubClient = new GitHubClient(this.onGitHubClientError);
this.junoClient = new JunoClient();
this.setIsNotificationConsoleExpanded = params?.setIsNotificationConsoleExpanded;
this.setNotificationConsoleData = params?.setNotificationConsoleData;
this.setInProgressConsoleDataIdToBeDeleted = params?.setInProgressConsoleDataIdToBeDeleted;
@@ -195,22 +172,9 @@ export default class Explorer {
});
this.isAccountReady = ko.observable<boolean>(false);
this._isInitializingNotebooks = false;
this.arcadiaToken = ko.observable<string>();
this.arcadiaToken.subscribe((token: string) => {
if (token) {
const notebookTabs = this.tabsManager.getTabs(ViewModels.CollectionTabKind.NotebookV2);
(notebookTabs || []).forEach((tab: NotebookV2Tab) => {
tab.reconfigureServiceEndpoints();
});
}
});
this.isShellEnabled = ko.observable(false);
this.isNotebooksEnabledForAccount = ko.observable(false);
this.isNotebooksEnabledForAccount.subscribe((isEnabledForAccount: boolean) => this.refreshCommandBarButtons());
this.isSparkEnabledForAccount = ko.observable(false);
this.isSparkEnabledForAccount.subscribe((isEnabledForAccount: boolean) => this.refreshCommandBarButtons());
this.hasStorageAnalyticsAfecFeature = ko.observable(false);
this.hasStorageAnalyticsAfecFeature.subscribe((enabled: boolean) => this.refreshCommandBarButtons());
this.isSynapseLinkUpdating = ko.observable<boolean>(false);
this.isAccountReady.subscribe(async (isAccountReady: boolean) => {
if (isAccountReady) {
@@ -219,63 +183,29 @@ export default class Explorer {
: this.refreshAllDatabases(true);
RouteHandler.getInstance().initHandler();
this.notebookWorkspaceManager = new NotebookWorkspaceManager();
this.arcadiaWorkspaces = ko.observableArray();
this._arcadiaManager = new ArcadiaResourceManager();
this._isAfecFeatureRegistered(Constants.AfecFeatures.StorageAnalytics).then((isRegistered) =>
this.hasStorageAnalyticsAfecFeature(isRegistered)
await this._refreshNotebooksEnabledStateForAccount();
this.isNotebookEnabled(
userContext.authType !== AuthType.ResourceToken &&
((await this._containsDefaultNotebookWorkspace(userContext.databaseAccount)) ||
userContext.features.enableNotebooks)
);
Promise.all([this._refreshNotebooksEnabledStateForAccount(), this._refreshSparkEnabledStateForAccount()]).then(
async () => {
this.isNotebookEnabled(
userContext.authType !== AuthType.ResourceToken &&
((await this._containsDefaultNotebookWorkspace(userContext.databaseAccount)) ||
userContext.features.enableNotebooks)
);
this.isShellEnabled(
this.isNotebookEnabled() &&
!userContext.databaseAccount.properties.isVirtualNetworkFilterEnabled &&
userContext.databaseAccount.properties.ipRules.length === 0
);
TelemetryProcessor.trace(Action.NotebookEnabled, ActionModifiers.Mark, {
isNotebookEnabled: this.isNotebookEnabled(),
dataExplorerArea: Constants.Areas.Notebook,
});
if (this.isNotebookEnabled()) {
await this.initNotebooks(userContext.databaseAccount);
const workspaces = await this._getArcadiaWorkspaces();
this.arcadiaWorkspaces(workspaces);
} else if (this.notebookToImport) {
// if notebooks is not enabled but the user is trying to do a quickstart setup with notebooks, open the SetupNotebooksPane
this._openSetupNotebooksPaneForQuickstart();
}
this.isSparkEnabled(
(this.isNotebookEnabled() &&
this.isSparkEnabledForAccount() &&
this.arcadiaWorkspaces() &&
this.arcadiaWorkspaces().length > 0) ||
userContext.features.enableSpark
);
if (this.isSparkEnabled()) {
trackEvent(
{ name: "LoadedWithSparkEnabled" },
{
subscriptionId: userContext.subscriptionId,
accountName: userContext.databaseAccount?.name,
accountId: userContext.databaseAccount?.id,
platform: configContext.platform,
}
);
const pollArcadiaTokenRefresh = async () => {
this.arcadiaToken(await this.getArcadiaToken());
setTimeout(() => pollArcadiaTokenRefresh(), this.getTokenRefreshInterval(this.arcadiaToken()));
};
await pollArcadiaTokenRefresh();
}
}
this.isShellEnabled(
this.isNotebookEnabled() &&
!userContext.databaseAccount.properties.isVirtualNetworkFilterEnabled &&
userContext.databaseAccount.properties.ipRules.length === 0
);
TelemetryProcessor.trace(Action.NotebookEnabled, ActionModifiers.Mark, {
isNotebookEnabled: this.isNotebookEnabled(),
dataExplorerArea: Constants.Areas.Notebook,
});
if (this.isNotebookEnabled()) {
await this.initNotebooks(userContext.databaseAccount);
} else if (this.notebookToImport) {
// if notebooks is not enabled but the user is trying to do a quickstart setup with notebooks, open the SetupNotebooksPane
this._openSetupNotebooksPaneForQuickstart();
}
}
});
this.memoryUsageInfo = ko.observable<DataModels.MemoryUsageInfo>();
@@ -286,11 +216,6 @@ export default class Explorer {
this.resourceTokenCollectionId = ko.observable<string>();
this.resourceTokenCollection = ko.observable<ViewModels.CollectionBase>();
this.resourceTokenPartitionKey = ko.observable<string>();
this.isMongoIndexingEnabled = ko.observable<boolean>(false);
this.isPublishNotebookPaneEnabled = ko.observable<boolean>(false);
this.canExceedMaximumValue = ko.computed<boolean>(() => userContext.features.canExceedMaximumValue);
this.isSchemaEnabled = ko.computed<boolean>(() => userContext.features.enableSchema);
this.databases = ko.observableArray<ViewModels.Database>();
@@ -445,8 +370,6 @@ export default class Explorer {
this.refreshNotebookList();
});
this.isSparkEnabled = ko.observable(false);
this.isSparkEnabled.subscribe((isEnabled: boolean) => this.refreshCommandBarButtons());
this.resourceTree = new ResourceTreeAdapter(this);
this.resourceTreeForResourceToken = new ResourceTreeAdapterForResourceToken(this);
this.notebookServerInfo = ko.observable<DataModels.NotebookWorkspaceConnectionInfo>({
@@ -490,23 +413,6 @@ export default class Explorer {
}
}
private onGitHubClientError = (error: any): void => {
Logger.logError(getErrorMessage(error), "NotebookManager/onGitHubClientError");
if (error.status === HttpStatusCodes.Unauthorized) {
this.gitHubOAuthService.resetToken();
this.showOkCancelModalDialog(
undefined,
"Cosmos DB cannot access your Github account anymore. Please connect to GitHub again.",
"Connect to GitHub",
() => this.openGitHubReposPanel("Connect to GitHub"),
"Cancel",
undefined
);
}
};
public openEnableSynapseLinkDialog(): void {
const addSynapseLinkDialogProps: DialogProps = {
linkProps: {
@@ -528,22 +434,17 @@ export default class Explorer {
this.isSynapseLinkUpdating(true);
useDialog.getState().closeDialog();
const resourceProviderClient = new ResourceProviderClientFactory().getOrCreate(userContext.databaseAccount.id);
try {
const databaseAccount: DataModels.DatabaseAccount = await resourceProviderClient.patchAsync(
userContext.databaseAccount.id,
"2019-12-12",
{
properties: {
enableAnalyticalStorage: true,
},
}
);
await update(userContext.subscriptionId, userContext.resourceGroup, userContext.databaseAccount.id, {
properties: {
enableAnalyticalStorage: true,
},
});
clearInProgressMessage();
logConsoleInfo("Enabled Azure Synapse Link for this account");
TelemetryProcessor.traceSuccess(Action.EnableAzureSynapseLink, {}, startTime);
updateUserContext({ databaseAccount });
userContext.databaseAccount.properties.enableAnalyticalStorage = true;
} catch (error) {
clearInProgressMessage();
logConsoleError(`Enabling Azure Synapse Link for this account failed. ${getErrorMessage(error)}`);
@@ -733,52 +634,6 @@ export default class Explorer {
window.open(Constants.Urls.feedbackEmail, "_blank");
};
public async getArcadiaToken(): Promise<string> {
return new Promise<string>((resolve: (token: string) => void, reject: (error: any) => void) => {
sendCachedDataMessage<string>(MessageTypes.GetArcadiaToken, undefined /** params **/).then(
(token: string) => {
resolve(token);
},
(error: any) => {
Logger.logError(getErrorMessage(error), "Explorer/getArcadiaToken");
resolve(undefined);
}
);
});
}
private async _getArcadiaWorkspaces(): Promise<ArcadiaWorkspaceItem[]> {
try {
const workspaces = await this._arcadiaManager.listWorkspacesAsync([userContext.subscriptionId]);
let workspaceItems: ArcadiaWorkspaceItem[] = new Array(workspaces.length);
const sparkPromises: Promise<void>[] = [];
workspaces.forEach((workspace, i) => {
let promise = this._arcadiaManager.listSparkPoolsAsync(workspaces[i].id).then(
(sparkpools) => {
workspaceItems[i] = { ...workspace, sparkPools: sparkpools };
},
(error) => {
Logger.logError(getErrorMessage(error), "Explorer/this._arcadiaManager.listSparkPoolsAsync");
}
);
sparkPromises.push(promise);
});
return Promise.all(sparkPromises).then(() => workspaceItems);
} catch (error) {
handleError(error, "Explorer/this._arcadiaManager.listWorkspacesAsync", "Get Arcadia workspaces failed");
return Promise.resolve([]);
}
}
public async createWorkspace(): Promise<string> {
return sendCachedDataMessage(MessageTypes.CreateWorkspace, undefined /** params **/);
}
public async createSparkPool(workspaceId: string): Promise<string> {
return sendCachedDataMessage(MessageTypes.CreateSparkPool, [workspaceId]);
}
public async initNotebooks(databaseAccount: DataModels.DatabaseAccount): Promise<void> {
if (!databaseAccount) {
throw new Error("No database account specified");
@@ -1178,7 +1033,6 @@ export default class Explorer {
onTakeSnapshot,
onClosePanel
);
this.isPublishNotebookPaneEnabled(true);
}
}
@@ -1457,57 +1311,6 @@ export default class Explorer {
}
}
public _refreshSparkEnabledStateForAccount = async (): Promise<void> => {
const { subscriptionId, authType } = userContext;
const armEndpoint = configContext.ARM_ENDPOINT;
if (!subscriptionId || !armEndpoint || authType === AuthType.EncryptedToken) {
// explorer is not aware of the database account yet
this.isSparkEnabledForAccount(false);
return;
}
const featureUri = `subscriptions/${subscriptionId}/providers/Microsoft.Features/providers/Microsoft.DocumentDb/features/${Constants.AfecFeatures.Spark}`;
const resourceProviderClient = new ResourceProviderClientFactory().getOrCreate(featureUri);
try {
const sparkNotebooksFeature: DataModels.AfecFeature = await resourceProviderClient.getAsync(
featureUri,
Constants.ArmApiVersions.armFeatures
);
const isEnabled =
(sparkNotebooksFeature &&
sparkNotebooksFeature.properties &&
sparkNotebooksFeature.properties.state === "Registered") ||
false;
this.isSparkEnabledForAccount(isEnabled);
} catch (error) {
Logger.logError(getErrorMessage(error), "Explorer/isSparkEnabledForAccount");
this.isSparkEnabledForAccount(false);
}
};
public _isAfecFeatureRegistered = async (featureName: string): Promise<boolean> => {
const { subscriptionId, authType } = userContext;
const armEndpoint = configContext.ARM_ENDPOINT;
if (!featureName || !subscriptionId || !armEndpoint || authType === AuthType.EncryptedToken) {
// explorer is not aware of the database account yet
return false;
}
const featureUri = `subscriptions/${subscriptionId}/providers/Microsoft.Features/providers/Microsoft.DocumentDb/features/${featureName}`;
const resourceProviderClient = new ResourceProviderClientFactory().getOrCreate(featureUri);
try {
const featureStatus: DataModels.AfecFeature = await resourceProviderClient.getAsync(
featureUri,
Constants.ArmApiVersions.armFeatures
);
const isEnabled =
(featureStatus && featureStatus.properties && featureStatus.properties.state === "Registered") || false;
return isEnabled;
} catch (error) {
Logger.logError(getErrorMessage(error), "Explorer/isSparkEnabledForAccount");
return false;
}
};
private refreshNotebookList = async (): Promise<void> => {
if (!this.isNotebookEnabled() || !this.notebookManager?.notebookContentClient) {
return;
@@ -1724,30 +1527,6 @@ export default class Explorer {
}
}
private getTokenRefreshInterval(token: string): number {
let tokenRefreshInterval = Constants.ClientDefaults.arcadiaTokenRefreshInterval;
if (!token) {
return tokenRefreshInterval;
}
try {
const tokenPayload = decryptJWTToken(this.arcadiaToken());
if (tokenPayload && tokenPayload.hasOwnProperty("exp")) {
const expirationTime = tokenPayload.exp as number; // seconds since unix epoch
const now = new Date().getTime() / 1000;
const tokenExpirationIntervalInMs = (expirationTime - now) * 1000;
if (tokenExpirationIntervalInMs < tokenRefreshInterval) {
tokenRefreshInterval =
tokenExpirationIntervalInMs - Constants.ClientDefaults.arcadiaTokenRefreshIntervalPaddingMs;
}
}
return tokenRefreshInterval;
} catch (error) {
Logger.logError(getErrorMessage(error), "Explorer/getTokenRefreshInterval");
return tokenRefreshInterval;
}
}
private _openSetupNotebooksPaneForQuickstart(): void {
const title = "Enable Notebooks (Preview)";
const description =
@@ -1779,11 +1558,6 @@ export default class Explorer {
}
}
public async loadSelectedDatabaseOffer(): Promise<void> {
const database = this.findSelectedDatabase();
await database?.loadOffer();
}
public async loadDatabaseOffers(): Promise<void> {
await Promise.all(
this.databases()?.map(async (database: ViewModels.Database) => {