Fix bug publish screenshot (#762)

[Preview this branch](https://cosmos-explorer-preview.azurewebsites.net/pull/762?feature.someFeatureFlagYouMightNeed=true) The main change in this PR fixes the snapshot functionality in the Publish pane-related components. Because the code cell outputs are now rendered in their own iframes for security reasons, a single snapshot of the notebook is no longer possible: each cell output takes its own snapshot and the snapshots are collated on the main notebook snapshot. - Move the snapshot functionality to notebook components: this removes the reference of the notebook DOM node that we must pass to the Publish pane via explorer. - Add slice in the state and actions in notebook redux for notebook snapshot requests and result - Add post robot message to take snapshots and receive results - Add logic in `NotebookRenderer` to wait for all output snapshots done before taking the main one collating. - Use `zustand` to share snapshot between Redux world and React world. This solves the issue of keeping the `PanelContainer` component generic, while being able to update its children (`PublishPanel` component) with the new snapshot. Additional changes: - Add `local()` in `@font-face` to check if font is already installed before downloading the font (must be done for Safari, but not Edge/Chrome) - Add "Export output to image" menu item in notebook cell, since each cell output can take its own snapshot (which can be downloaded) ![image](https://user-images.githubusercontent.com/21954022/117454706-b5f16600-af46-11eb-8535-6bf99f3d9170.png)
2025-12-20 01:11:25 +00:00 · 2021-05-11 20:24:05 +02:00
parent 4ed8fe9e7d
commit 861042c27e
24 changed files with 683 additions and 222 deletions
--- a/src/Explorer/Notebook/NotebookRenderer/NotebookRenderer.tsx
+++ b/src/Explorer/Notebook/NotebookRenderer/NotebookRenderer.tsx
@@ -1,6 +1,6 @@
 import { CellId } from "@nteract/commutable";
 import { CellType } from "@nteract/commutable/src";
-import { actions, ContentRef } from "@nteract/core";
+import { actions, ContentRef, selectors } from "@nteract/core";
 import { Cells, CodeCell, RawCell } from "@nteract/stateful-components";
 import MonacoEditor from "@nteract/stateful-components/lib/inputs/connected-editors/monacoEditor";
 import { PassedEditorProps } from "@nteract/stateful-components/lib/inputs/editor";
@@ -12,6 +12,8 @@ import { Dispatch } from "redux";
 import { userContext } from "../../../UserContext";
 import * as cdbActions from "../NotebookComponent/actions";
 import loadTransform from "../NotebookComponent/loadTransform";
+import { CdbAppState, SnapshotFragment, SnapshotRequest } from "../NotebookComponent/types";
+import { NotebookUtil } from "../NotebookUtil";
 import { AzureTheme } from "./AzureTheme";
 import "./base.css";
 import CellCreator from "./decorators/CellCreator";
@@ -32,10 +34,18 @@ export interface NotebookRendererBaseProps {
 }

 interface NotebookRendererDispatchProps {
-  updateNotebookParentDomElt: (contentRef: ContentRef, parentElt: HTMLElement) => void;
+  storeNotebookSnapshot: (imageSrc: string, requestId: string) => void;
+  notebookSnapshotError: (error: string) => void;
 }

-type NotebookRendererProps = NotebookRendererBaseProps & NotebookRendererDispatchProps;
+interface StateProps {
+  pendingSnapshotRequest: SnapshotRequest;
+  cellOutputSnapshots: Map<string, SnapshotFragment>;
+  notebookSnapshot: { imageSrc: string; requestId: string };
+  nbCodeCells: number;
+}
+
+type NotebookRendererProps = NotebookRendererBaseProps & NotebookRendererDispatchProps & StateProps;

 const decorate = (id: string, contentRef: ContentRef, cell_type: CellType, children: React.ReactNode) => {
  const Cell = () => (
@@ -60,27 +70,37 @@ const decorate = (id: string, contentRef: ContentRef, cell_type: CellType, child
 class BaseNotebookRenderer extends React.Component<NotebookRendererProps> {
  private notebookRendererRef = React.createRef<HTMLDivElement>();

-  constructor(props: NotebookRendererProps) {
-    super(props);
-
-    this.state = {
-      hoveredCellId: undefined,
-    };
-  }
-
  componentDidMount() {
    if (!userContext.features.sandboxNotebookOutputs) {
      loadTransform(this.props as any);
    }
-    this.props.updateNotebookParentDomElt(this.props.contentRef, this.notebookRendererRef.current);
  }

-  componentDidUpdate() {
-    this.props.updateNotebookParentDomElt(this.props.contentRef, this.notebookRendererRef.current);
-  }
-
-  componentWillUnmount() {
-    this.props.updateNotebookParentDomElt(this.props.contentRef, undefined);
+  async componentDidUpdate(): Promise<void> {
+    // Take a snapshot if there's a pending request and all the outputs are also saved
+    if (
+      this.props.pendingSnapshotRequest &&
+      this.props.pendingSnapshotRequest.type === "notebook" &&
+      this.props.pendingSnapshotRequest.notebookContentRef === this.props.contentRef &&
+      (!this.props.notebookSnapshot ||
+        this.props.pendingSnapshotRequest.requestId !== this.props.notebookSnapshot.requestId) &&
+      this.props.cellOutputSnapshots.size === this.props.nbCodeCells
+    ) {
+      try {
+        // Use Html2Canvas because it is much more reliable and fast than dom-to-file
+        const result = await NotebookUtil.takeScreenshotHtml2Canvas(
+          this.notebookRendererRef.current,
+          this.props.pendingSnapshotRequest.aspectRatio,
+          [...this.props.cellOutputSnapshots.values()],
+          this.props.pendingSnapshotRequest.downloadFilename
+        );
+        this.props.storeNotebookSnapshot(result.imageSrc, this.props.pendingSnapshotRequest.requestId);
+      } catch (error) {
+        this.props.notebookSnapshotError(error.message);
+      } finally {
+        this.setState({ processedSnapshotRequest: undefined });
+      }
+    }
  }

  render(): JSX.Element {
@@ -156,28 +176,40 @@ class BaseNotebookRenderer extends React.Component<NotebookRendererProps> {
  }
 }

+export const makeMapStateToProps = (
+  initialState: CdbAppState,
+  ownProps: NotebookRendererProps
+): ((state: CdbAppState) => StateProps) => {
+  const mapStateToProps = (state: CdbAppState): StateProps => {
+    const { contentRef } = ownProps;
+    const model = selectors.model(state, { contentRef });
+
+    let nbCodeCells;
+    if (model && model.type === "notebook") {
+      nbCodeCells = NotebookUtil.findCodeCellWithDisplay(model.notebook).length;
+    }
+    const { pendingSnapshotRequest, cellOutputSnapshots, notebookSnapshot } = state.cdb;
+    return { pendingSnapshotRequest, cellOutputSnapshots, notebookSnapshot, nbCodeCells };
+  };
+  return mapStateToProps;
+};
+
 const makeMapDispatchToProps = (initialDispatch: Dispatch, initialProps: NotebookRendererBaseProps) => {
  const mapDispatchToProps = (dispatch: Dispatch) => {
    return {
-      addTransform: (transform: React.ComponentType & { MIMETYPE: string }) => {
-        return dispatch(
+      addTransform: (transform: React.ComponentType & { MIMETYPE: string }) =>
+        dispatch(
          actions.addTransform({
            mediaType: transform.MIMETYPE,
            component: transform,
          })
-        );
-      },
-      updateNotebookParentDomElt: (contentRef: ContentRef, parentElt: HTMLElement) => {
-        return dispatch(
-          cdbActions.UpdateNotebookParentDomElt({
-            contentRef,
-            parentElt,
-          })
-        );
-      },
+        ),
+      storeNotebookSnapshot: (imageSrc: string, requestId: string) =>
+        dispatch(cdbActions.storeNotebookSnapshot({ imageSrc, requestId })),
+      notebookSnapshotError: (error: string) => dispatch(cdbActions.notebookSnapshotError({ error })),
    };
  };
  return mapDispatchToProps;
 };

-export default connect(null, makeMapDispatchToProps)(BaseNotebookRenderer);
+export default connect(makeMapStateToProps, makeMapDispatchToProps)(BaseNotebookRenderer);
--- a/src/Explorer/Notebook/NotebookRenderer/Toolbar.tsx
+++ b/src/Explorer/Notebook/NotebookRenderer/Toolbar.tsx
@@ -1,5 +1,5 @@
 import { ContextualMenuItemType, DirectionalHint, IconButton, IContextualMenuItem } from "@fluentui/react";
-import { CellId, CellType } from "@nteract/commutable";
+import { CellId, CellType, ImmutableCodeCell } from "@nteract/commutable";
 import { actions, AppState, DocumentRecordProps } from "@nteract/core";
 import * as selectors from "@nteract/selectors";
 import { CellToolbarContext } from "@nteract/stateful-components";
@@ -10,6 +10,8 @@ import { connect } from "react-redux";
 import { Dispatch } from "redux";
 import { Action, ActionModifiers } from "../../../Shared/Telemetry/TelemetryConstants";
 import * as cdbActions from "../NotebookComponent/actions";
+import { SnapshotRequest } from "../NotebookComponent/types";
+import { NotebookUtil } from "../NotebookUtil";

 export interface ComponentProps {
  contentRef: ContentRef;
@@ -26,12 +28,14 @@ interface DispatchProps {
  clearOutputs: () => void;
  deleteCell: () => void;
  traceNotebookTelemetry: (action: Action, actionModifier?: string, data?: any) => void;
+  takeNotebookSnapshot: (payload: SnapshotRequest) => void;
 }

 interface StateProps {
  cellType: CellType;
  cellIdAbove: CellId;
  cellIdBelow: CellId;
+  hasCodeOutput: boolean;
 }

 class BaseToolbar extends React.PureComponent<ComponentProps & DispatchProps & StateProps> {
@@ -58,11 +62,29 @@ class BaseToolbar extends React.PureComponent<ComponentProps & DispatchProps & S
            this.props.traceNotebookTelemetry(Action.NotebooksClearOutputsFromMenu, ActionModifiers.Mark);
          },
        },
-        {
-          key: "Divider",
-          itemType: ContextualMenuItemType.Divider,
-        },
      ]);
+
+      if (this.props.hasCodeOutput) {
+        items.push({
+          key: "Export output to image",
+          text: "Export output to image",
+          onClick: () => {
+            this.props.takeNotebookSnapshot({
+              requestId: new Date().getTime().toString(),
+              aspectRatio: undefined,
+              type: "celloutput",
+              cellId: this.props.id,
+              notebookContentRef: this.props.contentRef,
+              downloadFilename: `celloutput-${this.props.contentRef}_${this.props.id}.png`,
+            });
+          },
+        });
+      }
+
+      items.push({
+        key: "Divider",
+        itemType: ContextualMenuItemType.Divider,
+      });
    }

    items = items.concat([
@@ -183,12 +205,13 @@ const mapDispatchToProps = (
  deleteCell: () => dispatch(actions.deleteCell({ id, contentRef })),
  traceNotebookTelemetry: (action: Action, actionModifier?: string, data?: any) =>
    dispatch(cdbActions.traceNotebookTelemetry({ action, actionModifier, data })),
+  takeNotebookSnapshot: (request: SnapshotRequest) => dispatch(cdbActions.takeNotebookSnapshot(request)),
 });

 const makeMapStateToProps = (state: AppState, ownProps: ComponentProps): ((state: AppState) => StateProps) => {
  const mapStateToProps = (state: AppState) => {
-    const cellType = selectors.cell.cellFromState(state, { id: ownProps.id, contentRef: ownProps.contentRef })
-      .cell_type;
+    const cell = selectors.cell.cellFromState(state, { id: ownProps.id, contentRef: ownProps.contentRef });
+    const cellType = cell.cell_type;
    const model = selectors.model(state, { contentRef: ownProps.contentRef });
    const cellOrder = selectors.notebook.cellOrder(model as RecordOf<DocumentRecordProps>);
    const cellIndex = cellOrder.indexOf(ownProps.id);
@@ -199,6 +222,7 @@ const makeMapStateToProps = (state: AppState, ownProps: ComponentProps): ((state
      cellType,
      cellIdAbove,
      cellIdBelow,
+      hasCodeOutput: cellType === "code" && NotebookUtil.hasCodeCellOutput(cell as ImmutableCodeCell),
    };
  };
  return mapStateToProps;
--- a/src/Explorer/Notebook/NotebookRenderer/outputs/SandboxOutputs.tsx
+++ b/src/Explorer/Notebook/NotebookRenderer/outputs/SandboxOutputs.tsx
@@ -7,7 +7,9 @@ import postRobot from "post-robot";
 import React from "react";
 import { connect } from "react-redux";
 import { Dispatch } from "redux";
-import { CellOutputViewerProps } from "../../../../CellOutputViewer/CellOutputViewer";
+import { CellOutputViewerProps, SnapshotResponse } from "../../../../CellOutputViewer/CellOutputViewer";
+import * as cdbActions from "../../NotebookComponent/actions";
+import { CdbAppState, SnapshotFragment, SnapshotRequest } from "../../NotebookComponent/types";

 // Adapted from https://github.com/nteract/nteract/blob/main/packages/stateful-components/src/outputs/index.tsx
 // to add support for sandboxing using <iframe>
@@ -24,27 +26,47 @@ interface StateProps {
  expanded: boolean;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  outputs: Immutable.List<any>;
+
+  pendingSnapshotRequest: SnapshotRequest;
 }

 interface DispatchProps {
  onMetadataChange?: (metadata: JSONObject, mediaType: string, index?: number) => void;
+  storeNotebookSnapshot: (imageSrc: string, requestId: string) => void;
+  storeSnapshotFragment: (cellId: string, snapshotFragment: SnapshotFragment) => void;
+  notebookSnapshotError: (error: string) => void;
 }

-export class SandboxOutputs extends React.PureComponent<ComponentProps & StateProps & DispatchProps> {
+type SandboxOutputsProps = ComponentProps & StateProps & DispatchProps;
+
+export class SandboxOutputs extends React.Component<SandboxOutputsProps> {
  private childWindow: Window;
+  private nodeRef = React.createRef<HTMLDivElement>();
+
+  constructor(props: SandboxOutputsProps) {
+    super(props);
+
+    this.state = {
+      processedSnapshotRequest: undefined,
+    };
+  }

  render(): JSX.Element {
    // Using min-width to set the width of the iFrame, works around an issue in iOS that can prevent the iFrame from sizing correctly.
-    return (
-      <IframeResizer
-        checkOrigin={false}
-        loading="lazy"
-        heightCalculationMethod="taggedElement"
-        onLoad={(event) => this.handleFrameLoad(event)}
-        src="./cellOutputViewer.html"
-        style={{ height: "1px", width: "1px", minWidth: "100%", border: "none" }}
-        sandbox="allow-downloads allow-popups allow-forms allow-pointer-lock allow-scripts allow-popups-to-escape-sandbox"
-      />
+    return this.props.outputs && this.props.outputs.size > 0 ? (
+      <div ref={this.nodeRef}>
+        <IframeResizer
+          checkOrigin={false}
+          loading="lazy"
+          heightCalculationMethod="taggedElement"
+          onLoad={(event) => this.handleFrameLoad(event)}
+          src="./cellOutputViewer.html"
+          style={{ height: "1px", width: "1px", minWidth: "100%", border: "none" }}
+          sandbox="allow-downloads allow-popups allow-forms allow-pointer-lock allow-scripts allow-popups-to-escape-sandbox"
+        />
+      </div>
+    ) : (
+      <></>
    );
  }

@@ -76,8 +98,48 @@ export class SandboxOutputs extends React.PureComponent<ComponentProps & StatePr
    this.sendPropsToFrame();
  }

-  componentDidUpdate(): void {
+  async componentDidUpdate(prevProps: SandboxOutputsProps): Promise<void> {
    this.sendPropsToFrame();
+
+    if (
+      this.props.pendingSnapshotRequest &&
+      prevProps.pendingSnapshotRequest !== this.props.pendingSnapshotRequest &&
+      this.props.pendingSnapshotRequest.notebookContentRef === this.props.contentRef &&
+      this.nodeRef?.current
+    ) {
+      const boundingClientRect = this.nodeRef.current.getBoundingClientRect();
+
+      try {
+        const { data } = (await postRobot.send(
+          this.childWindow,
+          "snapshotRequest",
+          this.props.pendingSnapshotRequest
+        )) as { data: SnapshotResponse };
+        if (this.props.pendingSnapshotRequest.type === "notebook") {
+          if (data.imageSrc === undefined) {
+            this.props.storeSnapshotFragment(this.props.id, {
+              image: undefined,
+              boundingClientRect: boundingClientRect,
+              requestId: data.requestId,
+            });
+            return;
+          }
+          const image = new Image();
+          image.src = data.imageSrc;
+          image.onload = () => {
+            this.props.storeSnapshotFragment(this.props.id, {
+              image,
+              boundingClientRect: boundingClientRect,
+              requestId: data.requestId,
+            });
+          };
+        } else if (this.props.pendingSnapshotRequest.type === "celloutput") {
+          this.props.storeNotebookSnapshot(data.imageSrc, this.props.pendingSnapshotRequest.requestId);
+        }
+      } catch (error) {
+        this.props.notebookSnapshotError(error.message);
+      }
+    }
  }
 }

@@ -85,7 +147,7 @@ export const makeMapStateToProps = (
  initialState: AppState,
  ownProps: ComponentProps
 ): ((state: AppState) => StateProps) => {
-  const mapStateToProps = (state: AppState): StateProps => {
+  const mapStateToProps = (state: CdbAppState): StateProps => {
    let outputs = Immutable.List();
    let hidden = false;
    let expanded = false;
@@ -102,7 +164,17 @@ export const makeMapStateToProps = (
      }
    }

-    return { outputs, hidden, expanded };
+    // Determine whether to take a snapshot or not
+    let pendingSnapshotRequest = state.cdb.pendingSnapshotRequest;
+    if (
+      pendingSnapshotRequest &&
+      pendingSnapshotRequest.type === "celloutput" &&
+      pendingSnapshotRequest.cellId !== id
+    ) {
+      pendingSnapshotRequest = undefined;
+    }
+
+    return { outputs, hidden, expanded, pendingSnapshotRequest };
  };
  return mapStateToProps;
 };
@@ -125,6 +197,11 @@ export const makeMapDispatchToProps = (
          })
        );
      },
+      storeSnapshotFragment: (cellId: string, snapshot: SnapshotFragment) =>
+        dispatch(cdbActions.storeCellOutputSnapshot({ cellId, snapshot })),
+      storeNotebookSnapshot: (imageSrc: string, requestId: string) =>
+        dispatch(cdbActions.storeNotebookSnapshot({ imageSrc, requestId })),
+      notebookSnapshotError: (error: string) => dispatch(cdbActions.notebookSnapshotError({ error })),
    };
  };
  return mapDispatchToProps;