From aa8236666e1da7937684bb8b50b751ec2e038ee6 Mon Sep 17 00:00:00 2001 From: Tanuj Mittal Date: Fri, 5 Jun 2020 12:22:41 -0700 Subject: [PATCH] Use graphql in GitHubClient and misc fixes (#8) * Use graphql in GitHubClient * Replace usage of Array.find with _.find --- package-lock.json | 100 +-- package.json | 2 +- .../DialogReactComponent/DialogComponent.tsx | 3 +- .../DefaultDirectoryDropdownComponent.tsx | 3 +- .../Directory/DirectoryListComponent.tsx | 3 +- .../Controls/GitHub/AddRepoComponent.tsx | 2 +- .../Controls/GitHub/ReposListComponent.tsx | 11 +- src/Explorer/Explorer.ts | 2 +- .../Menus/CommandBar/CommandBarUtil.tsx | 6 +- .../Notebook/NotebookComponent/actions.ts | 19 - .../Notebook/NotebookComponent/epics.ts | 71 +- .../Notebook/NotebookComponent/reducers.ts | 5 - src/Explorer/Notebook/NotebookSamples.ts | 127 ++++ src/Explorer/OpenActions.ts | 14 +- src/Explorer/Panes/ClusterLibraryPane.ts | 3 +- src/Explorer/Panes/GitHubReposPane.ts | 39 +- src/Explorer/Tree/ResourceTreeAdapter.tsx | 20 +- src/GitHub/GitHubClient.test.ts | 142 ++-- src/GitHub/GitHubClient.ts | 657 ++++++++++-------- src/GitHub/GitHubContentProvider.test.ts | 42 +- src/GitHub/GitHubContentProvider.ts | 117 ++-- src/HostedExplorer.ts | 6 +- src/Utils/GitHubUtils.ts | 18 +- src/Utils/JunoUtils.ts | 6 +- 24 files changed, 761 insertions(+), 657 deletions(-) create mode 100644 src/Explorer/Notebook/NotebookSamples.ts diff --git a/package-lock.json b/package-lock.json index b49ef8b3f..c4993f67e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2422,32 +2422,42 @@ } }, "@octokit/auth-token": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.0.tgz", - "integrity": "sha512-eoOVMjILna7FVQf96iWc3+ZtE/ZT6y8ob8ZzcqKY1ibSQCnu4O/B7pJvzMx5cyZ/RjAff6DAdEb0O0Cjcxidkg==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.1.tgz", + "integrity": "sha512-NB81O5h39KfHYGtgfWr2booRxp2bWOJoqbWwbyUg2hw6h35ArWYlAST5B3XwAkbdcx13yt84hFXyFP5X0QToWA==", "requires": { - "@octokit/types": "^2.0.0" + "@octokit/types": "^4.0.1" + }, + "dependencies": { + "@octokit/types": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-4.0.2.tgz", + "integrity": "sha512-+4X6qfhT/fk/5FD66395NrFLxCzD6FsGlpPwfwvnukdyfYbhiZB/FJltiT1XM5Q63rGGBSf9FPaNV3WpNHm54A==", + "requires": { + "@types/node": ">= 8" + } + } } }, "@octokit/core": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-2.5.0.tgz", - "integrity": "sha512-uvzmkemQrBgD8xuGbjhxzJN1darJk9L2cS+M99cHrDG2jlSVpxNJVhoV86cXdYBqdHCc9Z995uLCczaaHIYA6Q==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-2.5.3.tgz", + "integrity": "sha512-23AHK9xBW0v79Ck8h5U+5iA4MW7aosqv+Yr6uZXolVGNzzHwryNH5wM386/6+etiKUTwLFZTqyMU9oQpIBZcFA==", "requires": { "@octokit/auth-token": "^2.4.0", "@octokit/graphql": "^4.3.1", "@octokit/request": "^5.4.0", - "@octokit/types": "^2.0.0", + "@octokit/types": "^4.0.1", "before-after-hook": "^2.1.0", "universal-user-agent": "^5.0.0" } }, "@octokit/endpoint": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.1.tgz", - "integrity": "sha512-pOPHaSz57SFT/m3R5P8MUu4wLPszokn5pXcB/pzavLTQf2jbU+6iayTvzaY6/BiotuRS0qyEUkx3QglT4U958A==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.2.tgz", + "integrity": "sha512-xs1mmCEZ2y4shXCpFjNq3UbmNR+bLzxtZim2L0zfEtj9R6O6kc4qLDvYw66hvO6lUsYzPTM5hMkltbuNAbRAcQ==", "requires": { - "@octokit/types": "^2.11.1", + "@octokit/types": "^4.0.1", "is-plain-object": "^3.0.0", "universal-user-agent": "^5.0.0" }, @@ -2468,21 +2478,21 @@ } }, "@octokit/graphql": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.4.0.tgz", - "integrity": "sha512-Du3hAaSROQ8EatmYoSAJjzAz3t79t9Opj/WY1zUgxVUGfIKn0AEjg+hlOLscF6fv6i/4y/CeUvsWgIfwMkTccw==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.0.tgz", + "integrity": "sha512-StJWfn0M1QfhL3NKBz31e1TdDNZrHLLS57J2hin92SIfzlOVBuUaRkp31AGkGOAFOAVtyEX6ZiZcsjcJDjeb5g==", "requires": { "@octokit/request": "^5.3.0", - "@octokit/types": "^2.0.0", + "@octokit/types": "^4.0.1", "universal-user-agent": "^5.0.0" } }, "@octokit/plugin-paginate-rest": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.2.0.tgz", - "integrity": "sha512-KoNxC3PLNar8UJwR+1VMQOw2IoOrrFdo5YOiDKnBhpVbKpw+zkBKNMNKwM44UWL25Vkn0Sl3nYIEGKY+gW5ebw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.2.1.tgz", + "integrity": "sha512-/tHpIF2XpN40AyhIq295YRjb4g7Q5eKob0qM3thYJ0Z+CgmNsWKM/fWse/SUR8+LdprP1O4ZzSKQE+71TCwK+w==", "requires": { - "@octokit/types": "^2.12.1" + "@octokit/types": "^4.0.1" } }, "@octokit/plugin-request-log": { @@ -2491,22 +2501,22 @@ "integrity": "sha512-ywoxP68aOT3zHCLgWZgwUJatiENeHE7xJzYjfz8WI0goynp96wETBF+d95b8g/uL4QmS6owPVlaxiz3wyMAzcw==" }, "@octokit/plugin-rest-endpoint-methods": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-3.7.1.tgz", - "integrity": "sha512-YOlcE3bbk2ohaOVdRj9ww7AUYfmnS9hwJJGSj3/rFlNfMGOId4G8dLlhghXpdNSn05H0FRoI94UlFUKnn30Cyw==", + "version": "3.12.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-3.12.3.tgz", + "integrity": "sha512-9nrVDP1tBd7EtobGr5hZcYGTM0kBNmIvPJazrUd5OJO0NZWiQaQOqAnzApmC9cZ4o7RempV21ScpWkKGhrT51A==", "requires": { - "@octokit/types": "^2.11.1", + "@octokit/types": "^4.0.0", "deprecation": "^2.3.1" } }, "@octokit/request": { - "version": "5.4.2", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.2.tgz", - "integrity": "sha512-zKdnGuQ2TQ2vFk9VU8awFT4+EYf92Z/v3OlzRaSh4RIP0H6cvW1BFPXq4XYvNez+TPQjqN+0uSkCYnMFFhcFrw==", + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.4.tgz", + "integrity": "sha512-vqv1lz41c6VTxUvF9nM+a6U+vvP3vGk7drDpr0DVQg4zyqlOiKVrY17DLD6de5okj+YLHKcoqaUZTBtlNZ1BtQ==", "requires": { "@octokit/endpoint": "^6.0.1", "@octokit/request-error": "^2.0.0", - "@octokit/types": "^2.11.1", + "@octokit/types": "^4.0.1", "deprecation": "^2.0.0", "is-plain-object": "^3.0.0", "node-fetch": "^2.3.0", @@ -2530,39 +2540,32 @@ } }, "@octokit/request-error": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.0.0.tgz", - "integrity": "sha512-rtYicB4Absc60rUv74Rjpzek84UbVHGHJRu4fNVlZ1mCcyUPPuzFfG9Rn6sjHrd95DEsmjSt1Axlc699ZlbDkw==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.0.1.tgz", + "integrity": "sha512-5lqBDJ9/TOehK82VvomQ6zFiZjPeSom8fLkFVLuYL3sKiIb5RB8iN/lenLkY7oBmyQcGP7FBMGiIZTO8jufaRQ==", "requires": { - "@octokit/types": "^2.0.0", + "@octokit/types": "^4.0.1", "deprecation": "^2.0.0", "once": "^1.4.0" } }, "@octokit/rest": { - "version": "17.5.1", - "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-17.5.1.tgz", - "integrity": "sha512-0rGY7eo0cw8FYX7jAtUgfy3j+05zhs9JvkPFegx00HAaayodM1ixlHhCOB5yirGbsVOxbRIWVkvKc2yY9367gg==", + "version": "17.9.2", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-17.9.2.tgz", + "integrity": "sha512-UXxiE0HhGQAPB3WDHTEu7lYMHH2uRcs/9f26XyHpGGiiXht8hgHWEk6fA7WglwwEvnj8V7mkJOgIntnij132UA==", "requires": { "@octokit/core": "^2.4.3", - "@octokit/plugin-paginate-rest": "^2.1.0", + "@octokit/plugin-paginate-rest": "^2.2.0", "@octokit/plugin-request-log": "^1.0.0", - "@octokit/plugin-rest-endpoint-methods": "3.7.1" + "@octokit/plugin-rest-endpoint-methods": "^3.12.2" } }, "@octokit/types": { - "version": "2.16.2", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-2.16.2.tgz", - "integrity": "sha512-O75k56TYvJ8WpAakWwYRN8Bgu60KrmX0z1KqFp1kNiFNkgW+JW+9EBKZ+S33PU6SLvbihqd+3drvPxKK68Ee8Q==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-4.0.2.tgz", + "integrity": "sha512-+4X6qfhT/fk/5FD66395NrFLxCzD6FsGlpPwfwvnukdyfYbhiZB/FJltiT1XM5Q63rGGBSf9FPaNV3WpNHm54A==", "requires": { "@types/node": ">= 8" - }, - "dependencies": { - "@types/node": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.1.tgz", - "integrity": "sha512-FAYBGwC+W6F9+huFIDtn43cpy7+SzG+atzRiTfdp3inUKL2hXnd4rG8hylJLIh4+hqrQy1P17kvJByE/z825hA==" - } } }, "@peculiar/asn1-schema": { @@ -3195,8 +3198,7 @@ "@types/node": { "version": "12.11.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.11.1.tgz", - "integrity": "sha512-TJtwsqZ39pqcljJpajeoofYRfeZ7/I/OMUQ5pR4q5wOKf2ocrUvBAZUMhWsOvKx3dVc/aaV5GluBivt0sWqA5A==", - "dev": true + "integrity": "sha512-TJtwsqZ39pqcljJpajeoofYRfeZ7/I/OMUQ5pR4q5wOKf2ocrUvBAZUMhWsOvKx3dVc/aaV5GluBivt0sWqA5A==" }, "@types/promise.prototype.finally": { "version": "2.0.3", diff --git a/package.json b/package.json index e73134687..2d47e59fb 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "@nteract/transform-plotly": "6.1.6", "@nteract/transform-vdom": "4.0.11", "@nteract/transform-vega": "7.0.6", - "@octokit/rest": "17.5.1", + "@octokit/rest": "17.9.2", "@phosphor/widgets": "1.9.3", "@uifabric/react-cards": "0.109.53", "@uifabric/styling": "7.11.2", diff --git a/src/Explorer/Controls/DialogReactComponent/DialogComponent.tsx b/src/Explorer/Controls/DialogReactComponent/DialogComponent.tsx index 18d22b640..0811dc24c 100644 --- a/src/Explorer/Controls/DialogReactComponent/DialogComponent.tsx +++ b/src/Explorer/Controls/DialogReactComponent/DialogComponent.tsx @@ -54,7 +54,8 @@ export class DialogComponent extends React.Component { styles: { title: { fontSize: DIALOG_TITLE_FONT_SIZE, fontWeight: DIALOG_TITLE_FONT_WEIGHT }, subText: { fontSize: DIALOG_SUBTEXT_FONT_SIZE } - } + }, + showCloseButton: false }, modalProps: { isBlocking: this.props.isModal }, minWidth: DIALOG_MIN_WIDTH, diff --git a/src/Explorer/Controls/Directory/DefaultDirectoryDropdownComponent.tsx b/src/Explorer/Controls/Directory/DefaultDirectoryDropdownComponent.tsx index dc6b72c3c..cd17aac9f 100644 --- a/src/Explorer/Controls/Directory/DefaultDirectoryDropdownComponent.tsx +++ b/src/Explorer/Controls/Directory/DefaultDirectoryDropdownComponent.tsx @@ -2,6 +2,7 @@ * React component for Switch Directory */ +import _ from "underscore"; import * as React from "react"; import { Dropdown, IDropdownOption, IDropdownProps } from "office-ui-fabric-react/lib/Dropdown"; import { Tenant } from "../../../Contracts/DataModels"; @@ -60,7 +61,7 @@ export class DefaultDirectoryDropdownComponent extends React.Component d.tenantId === option.key); + const selectedDirectory = _.find(this.props.directories, d => d.tenantId === option.key); if (!selectedDirectory) { return; } diff --git a/src/Explorer/Controls/Directory/DirectoryListComponent.tsx b/src/Explorer/Controls/Directory/DirectoryListComponent.tsx index 9ba6b4f27..633e16f60 100644 --- a/src/Explorer/Controls/Directory/DirectoryListComponent.tsx +++ b/src/Explorer/Controls/Directory/DirectoryListComponent.tsx @@ -1,3 +1,4 @@ +import _ from "underscore"; import * as React from "react"; import { DefaultButton, IButtonProps } from "office-ui-fabric-react/lib/Button"; @@ -114,7 +115,7 @@ export class DirectoryListComponent extends React.Component d.tenantId === selectedDirectoryId); + const selectedDirectory = _.find(this.props.directories, d => d.tenantId === selectedDirectoryId); this.props.onNewDirectorySelected(selectedDirectory); }; diff --git a/src/Explorer/Controls/GitHub/AddRepoComponent.tsx b/src/Explorer/Controls/GitHub/AddRepoComponent.tsx index 4d06a581b..c01b7693a 100644 --- a/src/Explorer/Controls/GitHub/AddRepoComponent.tsx +++ b/src/Explorer/Controls/GitHub/AddRepoComponent.tsx @@ -93,7 +93,7 @@ export class AddRepoComponent extends React.Component void; @@ -139,7 +140,7 @@ export class ReposListComponent extends React.Component } const checkboxProps: ICheckboxProps = { - ...ReposListComponent.getCheckboxPropsForLabel(GitHubUtils.toRepoFullName(item.repo.owner.login, item.repo.name)), + ...ReposListComponent.getCheckboxPropsForLabel(GitHubUtils.toRepoFullName(item.repo.owner, item.repo.name)), styles: ReposListCheckboxStyles, defaultChecked: true, onChange: () => this.props.unpinRepo(item) @@ -153,7 +154,7 @@ export class ReposListComponent extends React.Component return <>; } - const branchesProps = this.props.branchesProps[GitHubUtils.toRepoFullName(item.repo.owner.login, item.repo.name)]; + const branchesProps = this.props.branchesProps[GitHubUtils.toRepoFullName(item.repo.owner, item.repo.name)]; const options: IDropdownOption[] = branchesProps.branches.map(branch => ({ key: branch.name, text: branch.name, @@ -222,7 +223,7 @@ export class ReposListComponent extends React.Component private onRenderPinnedReposBranchesDropdownOption(option: IDropdownOption): JSX.Element { const item: RepoListItem = option.data; - const branchesProps = this.props.branchesProps[GitHubUtils.toRepoFullName(item.repo.owner.login, item.repo.name)]; + const branchesProps = this.props.branchesProps[GitHubUtils.toRepoFullName(item.repo.owner, item.repo.name)]; if (option.index === ReposListComponent.FooterIndex) { const linkProps: ILinkProps = { @@ -267,7 +268,7 @@ export class ReposListComponent extends React.Component } const checkboxProps: ICheckboxProps = { - ...ReposListComponent.getCheckboxPropsForLabel(GitHubUtils.toRepoFullName(item.repo.owner.login, item.repo.name)), + ...ReposListComponent.getCheckboxPropsForLabel(GitHubUtils.toRepoFullName(item.repo.owner, item.repo.name)), styles: ReposListCheckboxStyles, onChange: () => { const repoListItem = { ...item }; diff --git a/src/Explorer/Explorer.ts b/src/Explorer/Explorer.ts index 781c063b7..542909ee8 100644 --- a/src/Explorer/Explorer.ts +++ b/src/Explorer/Explorer.ts @@ -2583,7 +2583,7 @@ export default class Explorer implements ViewModels.Explorer { const item = NotebookUtil.createNotebookContentItem(name, path, "file"); const parent = this.resourceTree.myNotebooksContentRoot; - if (parent && this.isNotebookEnabled() && this.notebookClient) { + if (parent && parent.children && this.isNotebookEnabled() && this.notebookClient) { if (this._filePathToImportAndOpen === path) { this._filePathToImportAndOpen = null; // we don't want to try opening this path again } diff --git a/src/Explorer/Menus/CommandBar/CommandBarUtil.tsx b/src/Explorer/Menus/CommandBar/CommandBarUtil.tsx index 76bcd4514..7ade58739 100644 --- a/src/Explorer/Menus/CommandBar/CommandBarUtil.tsx +++ b/src/Explorer/Menus/CommandBar/CommandBarUtil.tsx @@ -1,3 +1,4 @@ +import _ from "underscore"; import * as React from "react"; import * as ViewModels from "../../../Contracts/ViewModels"; import { Observable } from "knockout"; @@ -46,7 +47,7 @@ export class CommandBarUtil { text: btn.commandButtonLabel || btn.tooltipText, "data-test": btn.commandButtonLabel || btn.tooltipText, title: btn.tooltipText, - name: "menuitem", + name: btn.commandButtonLabel || btn.tooltipText, disabled: btn.disabled, ariaLabel: btn.ariaLabel, buttonStyles: { @@ -126,6 +127,9 @@ export class CommandBarUtil { } if (btn.isDropdown) { + const selectedChild = _.find(btn.children, child => child.dropdownItemKey === btn.dropdownSelectedKey); + result.name = selectedChild?.commandButtonLabel || btn.dropdownPlaceholder; + const dropdownStyles: Partial = { root: { margin: 5 }, dropdown: { width: btn.dropdownWidth }, diff --git a/src/Explorer/Notebook/NotebookComponent/actions.ts b/src/Explorer/Notebook/NotebookComponent/actions.ts index cfc1c763f..c081f3003 100644 --- a/src/Explorer/Notebook/NotebookComponent/actions.ts +++ b/src/Explorer/Notebook/NotebookComponent/actions.ts @@ -17,25 +17,6 @@ export const closeNotebook = (payload: { contentRef: ContentRef }): CloseNoteboo }; }; -export const UPDATE_LAST_MODIFIED = "UPDATE_LAST_MODIFIED"; -export interface UpdateLastModifiedAction { - type: "UPDATE_LAST_MODIFIED"; - payload: { - contentRef: ContentRef; - lastModified: string; - }; -} - -export const updateLastModified = (payload: { - contentRef: ContentRef; - lastModified: string; -}): UpdateLastModifiedAction => { - return { - type: UPDATE_LAST_MODIFIED, - payload - }; -}; - export const EXECUTE_FOCUSED_CELL_AND_FOCUS_NEXT = "EXECUTE_FOCUSED_CELL_AND_FOCUS_NEXT"; export interface ExecuteFocusedCellAndFocusNextAction { type: "EXECUTE_FOCUSED_CELL_AND_FOCUS_NEXT"; diff --git a/src/Explorer/Notebook/NotebookComponent/epics.ts b/src/Explorer/Notebook/NotebookComponent/epics.ts index d22621be8..73dc59d62 100644 --- a/src/Explorer/Notebook/NotebookComponent/epics.ts +++ b/src/Explorer/Notebook/NotebookComponent/epics.ts @@ -1,4 +1,4 @@ -import { empty, merge, of, timer, interval, concat, Subject, Subscriber, Observable, Observer } from "rxjs"; +import { empty, merge, of, timer, concat, Subject, Subscriber, Observable, Observer } from "rxjs"; import { webSocket } from "rxjs/webSocket"; import { ActionsObservable, StateObservable } from "redux-observable"; import { ofType } from "redux-observable"; @@ -10,7 +10,6 @@ import { map, switchMap, take, - distinctUntilChanged, filter, catchError, first, @@ -21,7 +20,6 @@ import { AppState, ServerConfig as JupyterServerConfig, JupyterHostRecordProps, - JupyterHostRecord, RemoteKernelProps, castToSessionId, createKernelRef, @@ -29,8 +27,7 @@ import { ContentRef, KernelInfo, actions, - selectors, - IContentProvider + selectors } from "@nteract/core"; import { message, JupyterMessage, Channels, createMessage, childOf, ofMessageType } from "@nteract/messaging"; import { sessions, kernels } from "rx-jupyter"; @@ -752,69 +749,6 @@ export const cleanKernelOnConnectionLostEpic = ( ); }; -/** - * Workaround for issue: https://github.com/nteract/nteract/issues/4583 - * We reajust the property - * @param action$ - * @param state$ - */ -const adjustLastModifiedOnSaveEpic = ( - action$: ActionsObservable, - state$: StateObservable, - dependencies: { contentProvider: IContentProvider } -): Observable<{} | CdbActions.UpdateLastModifiedAction> => { - return action$.pipe( - ofType(actions.SAVE_FULFILLED), - mergeMap(action => { - const pollDelayMs = 500; - const nbAttempts = 4; - - // Retry updating last modified - const currentHost = selectors.currentHost(state$.value); - const serverConfig = selectors.serverConfig(currentHost as JupyterHostRecord); - const filepath = selectors.filepath(state$.value, { contentRef: action.payload.contentRef }); - const content = selectors.content(state$.value, { contentRef: action.payload.contentRef }); - const lastSaved = (content.lastSaved as any) as string; - const contentProvider = dependencies.contentProvider; - - // Query until value is stable - return interval(pollDelayMs) - .pipe(take(nbAttempts)) - .pipe( - mergeMap(x => - contentProvider.get(serverConfig, filepath, { content: 0 }).pipe( - map(xhr => { - if (xhr.status !== 200 || typeof xhr.response === "string") { - return undefined; - } - const model = xhr.response; - const lastModified = model.last_modified; - if (lastModified === lastSaved) { - return undefined; - } - // Return last modified - return lastModified; - }) - ) - ), - distinctUntilChanged(), - mergeMap(lastModified => { - if (!lastModified) { - return empty(); - } - - return of( - CdbActions.updateLastModified({ - contentRef: action.payload.contentRef, - lastModified - }) - ); - }) - ); - }) - ); -}; - /** * Execute focused cell and focus next cell * @param action$ @@ -917,7 +851,6 @@ export const allEpics = [ acquireKernelInfoEpic, handleKernelConnectionLostEpic, cleanKernelOnConnectionLostEpic, - adjustLastModifiedOnSaveEpic, executeFocusedCellAndFocusNextEpic, closeUnsupportedMimetypesEpic, closeContentFailedToFetchEpic, diff --git a/src/Explorer/Notebook/NotebookComponent/reducers.ts b/src/Explorer/Notebook/NotebookComponent/reducers.ts index 0feb63b42..dec2fab1e 100644 --- a/src/Explorer/Notebook/NotebookComponent/reducers.ts +++ b/src/Explorer/Notebook/NotebookComponent/reducers.ts @@ -51,11 +51,6 @@ export const coreReducer = (state: CoreRecord, action: Action) => { .setIn(path.concat("displayName"), kernelspecs.displayName) .setIn(path.concat("language"), kernelspecs.language); } - case cdbActions.UPDATE_LAST_MODIFIED: { - typedAction = action as cdbActions.UpdateLastModifiedAction; - const path = ["entities", "contents", "byRef", typedAction.payload.contentRef, "lastSaved"]; - return state.setIn(path, typedAction.payload.lastModified); - } default: return nteractReducers.core(state as any, action as any); } diff --git a/src/Explorer/Notebook/NotebookSamples.ts b/src/Explorer/Notebook/NotebookSamples.ts new file mode 100644 index 000000000..49988326c --- /dev/null +++ b/src/Explorer/Notebook/NotebookSamples.ts @@ -0,0 +1,127 @@ +import { IGitHubRepo, IGitHubBranch } from "../../GitHub/GitHubClient"; + +export const SamplesRepo: IGitHubRepo = { + name: "cosmos-notebooks", + owner: "Azure-Samples", + private: false +}; + +export const SamplesBranch: IGitHubBranch = { + name: "master" +}; + +export const isSamplesCall = (owner: string, repo: string, branch?: string): boolean => { + return owner === SamplesRepo.owner && repo === SamplesRepo.name && (!branch || branch === SamplesBranch.name); +}; + +// GitHub API calls have a rate limit of 5000 requests per hour. So if we get high traffic on Data Explorer +// loading samples exceed that limit. Using this hard coded response for samples until we fix that. +export const SamplesContentsQueryResponse = { + repository: { + owner: { + login: "Azure-Samples" + }, + name: "cosmos-notebooks", + isPrivate: false, + ref: { + name: "master", + target: { + history: { + nodes: [ + { + oid: "cda7facb9e039b173f3376200c26c859896e7974", + message: + "Merge pull request #45 from Azure-Samples/users/deborahc/pythonSampleUpdates\n\nAdd bokeh version to notebook", + committer: { + date: "2020-05-28T11:28:01-07:00" + } + } + ] + } + } + }, + object: { + entries: [ + { + name: ".github", + type: "tree", + object: {} + }, + { + name: ".gitignore", + type: "blob", + object: { + oid: "3e759b75bf455ac809d0987d369aab89137b5689", + byteSize: 5582 + } + }, + { + name: "1. GettingStarted.ipynb", + type: "blob", + object: { + oid: "0732ff5366e4aefdc4c378c61cbd968664f0acec", + byteSize: 3933 + } + }, + { + name: "2. Visualization.ipynb", + type: "blob", + object: { + oid: "6b16b0740a77afdd38a95bc6c3ebd0f2f17d9465", + byteSize: 820317 + } + }, + { + name: "3. RequestUnits.ipynb", + type: "blob", + object: { + oid: "252b79a4adc81e9f2ffde453231b695d75e270e8", + byteSize: 9490 + } + }, + { + name: "4. Indexing.ipynb", + type: "blob", + object: { + oid: "e10dd67bd1c55c345226769e4f80e43659ef9cd5", + byteSize: 10394 + } + }, + { + name: "5. StoredProcedures.ipynb", + type: "blob", + object: { + oid: "949941949920de4d2d111149e2182e9657cc8134", + byteSize: 11818 + } + }, + { + name: "6. GlobalDistribution.ipynb", + type: "blob", + object: { + oid: "b91c31dacacbc9e35750d9054063dda4a5309f3b", + byteSize: 11375 + } + }, + { + name: "7. IoTAnomalyDetection.ipynb", + type: "blob", + object: { + oid: "82057ae52a67721a5966e2361317f5dfbd0ee595", + byteSize: 377939 + } + }, + { + name: "All_API_quickstarts", + type: "tree", + object: {} + }, + { + name: "CSharp_quickstarts", + type: "tree", + object: {} + } + ] + } + } +}; diff --git a/src/Explorer/OpenActions.ts b/src/Explorer/OpenActions.ts index b2bd2865e..a692d09a4 100644 --- a/src/Explorer/OpenActions.ts +++ b/src/Explorer/OpenActions.ts @@ -155,19 +155,7 @@ function openPane(action: ActionContracts.OpenPane, explorer: ViewModels.Explore } function openFile(action: ActionContracts.OpenSampleNotebook, explorer: ViewModels.Explorer) { - let path: string; - if (action.hasOwnProperty("file")) { - // This is deprecated - const downloadUrl: string = (action as any).file.download_url; - path = downloadUrl.replace( - "raw.githubusercontent.com/Azure-Samples/cosmos-notebooks", - "github.com/Azure-Samples/cosmos-notebooks/blob" - ); // convert raw download url to something which GitHubContentProvider understands - } else { - path = action.path; - } - - explorer.handleOpenFileAction(path); + explorer.handleOpenFileAction(decodeURIComponent(action.path)); } function generateQueryText(action: ActionContracts.OpenQueryTab, partitionKeyProperty: string): string { diff --git a/src/Explorer/Panes/ClusterLibraryPane.ts b/src/Explorer/Panes/ClusterLibraryPane.ts index 6e5b99b05..3a9078e04 100644 --- a/src/Explorer/Panes/ClusterLibraryPane.ts +++ b/src/Explorer/Panes/ClusterLibraryPane.ts @@ -1,3 +1,4 @@ +import _ from "underscore"; import * as ko from "knockout"; import * as Constants from "../../Common/Constants"; import * as ViewModels from "../../Contracts/ViewModels"; @@ -81,7 +82,7 @@ export class ClusterLibraryPane extends ContextualPaneBase { private _onInstalledChanged = (libraryName: string, installed: boolean): void => { const items = this._clusterLibraryProps().libraryItems; - const library = items.find(item => item.name === libraryName); + const library = _.find(items, item => item.name === libraryName); library.installed = installed; this._clusterLibraryProps.valueHasMutated(); }; diff --git a/src/Explorer/Panes/GitHubReposPane.ts b/src/Explorer/Panes/GitHubReposPane.ts index 93a5df309..e310c3ea8 100644 --- a/src/Explorer/Panes/GitHubReposPane.ts +++ b/src/Explorer/Panes/GitHubReposPane.ts @@ -1,19 +1,20 @@ +import _ from "underscore"; import { Areas, HttpStatusCodes } from "../../Common/Constants"; import { Logger } from "../../Common/Logger"; import * as ViewModels from "../../Contracts/ViewModels"; -import { GitHubClient, IGitHubRepo } from "../../GitHub/GitHubClient"; +import { GitHubClient, IGitHubPageInfo, IGitHubRepo } from "../../GitHub/GitHubClient"; import { IPinnedRepo, JunoClient } from "../../Juno/JunoClient"; import { Action, ActionModifiers } from "../../Shared/Telemetry/TelemetryConstants"; import TelemetryProcessor from "../../Shared/Telemetry/TelemetryProcessor"; import { GitHubUtils } from "../../Utils/GitHubUtils"; +import { JunoUtils } from "../../Utils/JunoUtils"; import { NotificationConsoleUtils } from "../../Utils/NotificationConsoleUtils"; import { AuthorizeAccessComponent } from "../Controls/GitHub/AuthorizeAccessComponent"; -import { GitHubReposComponentProps, RepoListItem, GitHubReposComponent } from "../Controls/GitHub/GitHubReposComponent"; +import { GitHubReposComponent, GitHubReposComponentProps, RepoListItem } from "../Controls/GitHub/GitHubReposComponent"; import { GitHubReposComponentAdapter } from "../Controls/GitHub/GitHubReposComponentAdapter"; import { BranchesProps, PinnedReposProps, UnpinnedReposProps } from "../Controls/GitHub/ReposListComponent"; import { ConsoleDataType } from "../Menus/NotificationConsole/NotificationConsoleComponent"; import { ContextualPaneBase } from "./ContextualPaneBase"; -import { JunoUtils } from "../../Utils/JunoUtils"; export class GitHubReposPane extends ContextualPaneBase { private static readonly PageSize = 30; @@ -29,6 +30,7 @@ export class GitHubReposPane extends ContextualPaneBase { private gitHubReposAdapter: GitHubReposComponentAdapter; private allGitHubRepos: IGitHubRepo[]; + private allGitHubReposLastPageInfo?: IGitHubPageInfo; private pinnedReposUpdated: boolean; constructor(options: ViewModels.GitHubReposPaneOptions) { @@ -73,6 +75,7 @@ export class GitHubReposPane extends ContextualPaneBase { this.gitHubReposAdapter = new GitHubReposComponentAdapter(this.gitHubReposProps); this.allGitHubRepos = []; + this.allGitHubReposLastPageInfo = undefined; this.pinnedReposUpdated = false; } @@ -115,6 +118,7 @@ export class GitHubReposPane extends ContextualPaneBase { // Reset cached repos this.allGitHubRepos = []; + this.allGitHubReposLastPageInfo = undefined; // Reset flags this.pinnedReposUpdated = false; @@ -164,29 +168,28 @@ export class GitHubReposPane extends ContextualPaneBase { const unpinnedGitHubRepos = this.allGitHubRepos.filter( gitHubRepo => this.pinnedReposProps.repos.findIndex( - pinnedRepo => pinnedRepo.key === GitHubUtils.toRepoFullName(gitHubRepo.owner.login, gitHubRepo.name) + pinnedRepo => pinnedRepo.key === GitHubUtils.toRepoFullName(gitHubRepo.owner, gitHubRepo.name) ) === -1 ); return unpinnedGitHubRepos.map(gitHubRepo => ({ - key: GitHubUtils.toRepoFullName(gitHubRepo.owner.login, gitHubRepo.name), + key: GitHubUtils.toRepoFullName(gitHubRepo.owner, gitHubRepo.name), repo: gitHubRepo, branches: [] })); } private async loadMoreBranches(repo: IGitHubRepo): Promise { - const branchesProps = this.branchesProps[GitHubUtils.toRepoFullName(repo.owner.login, repo.name)]; + const branchesProps = this.branchesProps[GitHubUtils.toRepoFullName(repo.owner, repo.name)]; branchesProps.hasMore = true; branchesProps.isLoading = true; this.triggerRender(); - const nextPage = Math.floor(branchesProps.branches.length / GitHubReposPane.PageSize) + 1; try { const response = await this.gitHubClient.getBranchesAsync( - repo.owner.login, + repo.owner, repo.name, - nextPage, - GitHubReposPane.PageSize + GitHubReposPane.PageSize, + branchesProps.lastPageInfo?.endCursor ); if (response.status !== HttpStatusCodes.OK) { throw new Error(`Received HTTP ${response.status} when fetching branches`); @@ -194,6 +197,7 @@ export class GitHubReposPane extends ContextualPaneBase { if (response.data) { branchesProps.branches = branchesProps.branches.concat(response.data); + branchesProps.lastPageInfo = response.pageInfo; } } catch (error) { const message = `Failed to fetch branches: ${error}`; @@ -202,7 +206,7 @@ export class GitHubReposPane extends ContextualPaneBase { } branchesProps.isLoading = false; - branchesProps.hasMore = branchesProps.branches.length === GitHubReposPane.PageSize * nextPage; + branchesProps.hasMore = branchesProps.lastPageInfo?.hasNextPage; this.triggerRender(); } @@ -211,15 +215,18 @@ export class GitHubReposPane extends ContextualPaneBase { this.unpinnedReposProps.hasMore = true; this.triggerRender(); - const nextPage = Math.floor(this.allGitHubRepos.length / GitHubReposPane.PageSize) + 1; try { - const response = await this.gitHubClient.getReposAsync(nextPage, GitHubReposPane.PageSize); + const response = await this.gitHubClient.getReposAsync( + GitHubReposPane.PageSize, + this.allGitHubReposLastPageInfo?.endCursor + ); if (response.status !== HttpStatusCodes.OK) { throw new Error(`Received HTTP ${response.status} when fetching unpinned repos`); } if (response.data) { this.allGitHubRepos = this.allGitHubRepos.concat(response.data); + this.allGitHubReposLastPageInfo = response.pageInfo; this.unpinnedReposProps.repos = this.calculateUnpinnedRepos(); } } catch (error) { @@ -229,7 +236,7 @@ export class GitHubReposPane extends ContextualPaneBase { } this.unpinnedReposProps.isLoading = false; - this.unpinnedReposProps.hasMore = this.allGitHubRepos.length === GitHubReposPane.PageSize * nextPage; + this.unpinnedReposProps.hasMore = this.allGitHubReposLastPageInfo?.hasNextPage; this.triggerRender(); } @@ -253,7 +260,7 @@ export class GitHubReposPane extends ContextualPaneBase { this.pinnedReposUpdated = true; const initialReposLength = this.pinnedReposProps.repos.length; - const existingRepo = this.pinnedReposProps.repos.find(repo => repo.key === item.key); + const existingRepo = _.find(this.pinnedReposProps.repos, repo => repo.key === item.key); if (existingRepo) { existingRepo.branches = item.branches; } else { @@ -318,6 +325,7 @@ export class GitHubReposPane extends ContextualPaneBase { if (!this.branchesProps[item.key]) { this.branchesProps[item.key] = { branches: [], + lastPageInfo: undefined, hasMore: true, isLoading: true, loadMore: (): Promise => this.loadMoreBranches(item.repo) @@ -329,6 +337,7 @@ export class GitHubReposPane extends ContextualPaneBase { private async refreshUnpinnedRepoListItems(): Promise { this.allGitHubRepos = []; + this.allGitHubReposLastPageInfo = undefined; this.unpinnedReposProps.repos = []; this.loadMoreUnpinnedRepos(); } diff --git a/src/Explorer/Tree/ResourceTreeAdapter.tsx b/src/Explorer/Tree/ResourceTreeAdapter.tsx index a1ce31fda..881eab560 100644 --- a/src/Explorer/Tree/ResourceTreeAdapter.tsx +++ b/src/Explorer/Tree/ResourceTreeAdapter.tsx @@ -14,7 +14,6 @@ import CollectionIcon from "../../../images/tree-collection.svg"; import DeleteIcon from "../../../images/delete.svg"; import NotebookIcon from "../../../images/notebook/Notebook-resource.svg"; import RefreshIcon from "../../../images/refresh-cosmos.svg"; -import { IGitHubRepo, IGitHubBranch } from "../../GitHub/GitHubClient"; import NewNotebookIcon from "../../../images/notebook/Notebook-new.svg"; import FileIcon from "../../../images/notebook/file-cosmos.svg"; import { ArrayHashMap } from "../../Common/ArrayHashMap"; @@ -26,21 +25,11 @@ import TelemetryProcessor from "../../Shared/Telemetry/TelemetryProcessor"; import { Action, ActionModifiers } from "../../Shared/Telemetry/TelemetryConstants"; import { Areas } from "../../Common/Constants"; import { GitHubUtils } from "../../Utils/GitHubUtils"; +import { SamplesRepo, SamplesBranch } from "../Notebook/NotebookSamples"; export class ResourceTreeAdapter implements ReactAdapter { private static readonly DataTitle = "DATA"; private static readonly NotebooksTitle = "NOTEBOOKS"; - - private static readonly SamplesRepo: IGitHubRepo = { - name: "cosmos-notebooks", - owner: { - login: "Azure-Samples" - }, - private: false - }; - private static readonly SamplesBranch: IGitHubBranch = { - name: "master" - }; private static readonly PseudoDirPath = "PsuedoDir"; public parameters: ko.Observable; @@ -103,12 +92,7 @@ export class ResourceTreeAdapter implements ReactAdapter { this.sampleNotebooksContentRoot = { name: "Sample Notebooks (View Only)", - path: GitHubUtils.toContentUri( - ResourceTreeAdapter.SamplesRepo.owner.login, - ResourceTreeAdapter.SamplesRepo.name, - ResourceTreeAdapter.SamplesBranch.name, - "" - ), + path: GitHubUtils.toContentUri(SamplesRepo.owner, SamplesRepo.name, SamplesBranch.name, ""), type: NotebookContentItemType.Directory }; refreshTasks.push( diff --git a/src/GitHub/GitHubClient.test.ts b/src/GitHub/GitHubClient.test.ts index 8fa90f00f..76a24bd93 100644 --- a/src/GitHub/GitHubClient.test.ts +++ b/src/GitHub/GitHubClient.test.ts @@ -1,92 +1,110 @@ -import ko from "knockout"; import { HttpStatusCodes } from "../Common/Constants"; -import { GitHubClient, IGitHubBranch, IGitHubRepo } from "./GitHubClient"; +import { GitHubClient, IGitHubFile } from "./GitHubClient"; +import { SamplesRepo, SamplesBranch, SamplesContentsQueryResponse } from "../Explorer/Notebook/NotebookSamples"; const invalidTokenCallback = jest.fn(); -// Use a dummy token to get around API rate limit (same as AZURESAMPLESCOSMOSDBPAT in webpack.config.js) -const gitHubClient = new GitHubClient("99e38770e29b4a61d7c49f188780504efd35cc86", invalidTokenCallback); -const samplesRepo: IGitHubRepo = { - name: "cosmos-notebooks", - owner: { - login: "Azure-Samples" - }, - private: false -}; -const samplesBranch: IGitHubBranch = { - name: "master" -}; -const sampleFilePath = ".gitignore"; -const sampleDirPath = ".github"; +// Use a dummy token to get around API rate limit (something which doesn't affect the API quota for AZURESAMPLESCOSMOSDBPAT in Config.ts) +const gitHubClient = new GitHubClient("cd1906b9534362fab6ce45d6db6c76b59e55bc50", invalidTokenCallback); -describe.skip("GitHubClient", () => { +const validateGitHubFile = (file: IGitHubFile) => { + expect(file.branch).toEqual(SamplesBranch); + expect(file.commit).toBeDefined(); + expect(file.name).toBeDefined(); + expect(file.path).toBeDefined(); + expect(file.repo).toEqual(SamplesRepo); + expect(file.type).toBeDefined(); + + switch (file.type) { + case "blob": + expect(file.sha).toBeDefined(); + expect(file.size).toBeDefined(); + break; + + case "tree": + expect(file.sha).toBeUndefined(); + expect(file.size).toBeUndefined(); + break; + + default: + throw new Error(`Unsupported github file type: ${file.type}`); + } +}; + +describe("GitHubClient", () => { it("getRepoAsync returns valid repo", async () => { - const response = await gitHubClient.getRepoAsync(samplesRepo.owner.login, samplesRepo.name); - expect(response.status).toBe(HttpStatusCodes.OK); - expect(response.data.name).toBe(samplesRepo.name); - expect(response.data.owner.login).toBe(samplesRepo.owner.login); + const response = await gitHubClient.getRepoAsync(SamplesRepo.owner, SamplesRepo.name); + expect(response).toEqual({ + status: HttpStatusCodes.OK, + data: SamplesRepo + }); }); it("getReposAsync returns repos for authenticated user", async () => { - const response = await gitHubClient.getReposAsync(1, 1); + const response = await gitHubClient.getReposAsync(1); expect(response.status).toBe(HttpStatusCodes.OK); + expect(response.data).toBeDefined(); expect(response.data.length).toBe(1); + expect(response.pageInfo).toBeDefined(); }); it("getBranchesAsync returns branches for a repo", async () => { - const response = await gitHubClient.getBranchesAsync(samplesRepo.owner.login, samplesRepo.name, 1, 1); + const response = await gitHubClient.getBranchesAsync(SamplesRepo.owner, SamplesRepo.name, 1); expect(response.status).toBe(HttpStatusCodes.OK); - expect(response.data.length).toBe(1); + expect(response.data).toEqual([SamplesBranch]); + expect(response.pageInfo).toBeDefined(); }); - it("getCommitsAsync returns commits for a file", async () => { - const response = await gitHubClient.getCommitsAsync( - samplesRepo.owner.login, - samplesRepo.name, - samplesBranch.name, - sampleFilePath, - 1, - 1 - ); + it("getContentsAsync returns files in the repo", async () => { + const response = await gitHubClient.getContentsAsync(SamplesRepo.owner, SamplesRepo.name, SamplesBranch.name); expect(response.status).toBe(HttpStatusCodes.OK); - expect(response.data.length).toBe(1); + expect(response.data).toBeDefined(); + + const data = response.data as IGitHubFile[]; + expect(data.length).toBeGreaterThan(0); + data.forEach(content => validateGitHubFile(content)); }); - it("getDirContentsAsync returns files in the repo", async () => { - const response = await gitHubClient.getDirContentsAsync( - samplesRepo.owner.login, - samplesRepo.name, - samplesBranch.name, - "" + it("getContentsAsync returns files in a dir", async () => { + const samplesDir = SamplesContentsQueryResponse.repository.object.entries.find(file => file.type === "tree"); + const response = await gitHubClient.getContentsAsync( + SamplesRepo.owner, + SamplesRepo.name, + SamplesBranch.name, + samplesDir.name ); + expect(response.status).toBe(HttpStatusCodes.OK); - expect(response.data.length).toBeGreaterThan(0); - expect(response.data[0].repo).toEqual(samplesRepo); - expect(response.data[0].branch).toEqual(samplesBranch); + expect(response.data).toBeDefined(); + + const data = response.data as IGitHubFile[]; + expect(data.length).toBeGreaterThan(0); + data.forEach(content => validateGitHubFile(content)); }); - it("getDirContentsAsync returns files in a dir", async () => { - const response = await gitHubClient.getDirContentsAsync( - samplesRepo.owner.login, - samplesRepo.name, - samplesBranch.name, - sampleDirPath + it("getContentsAsync returns a file", async () => { + const samplesFile = SamplesContentsQueryResponse.repository.object.entries.find(file => file.type === "blob"); + const response = await gitHubClient.getContentsAsync( + SamplesRepo.owner, + SamplesRepo.name, + SamplesBranch.name, + samplesFile.name ); + expect(response.status).toBe(HttpStatusCodes.OK); - expect(response.data.length).toBeGreaterThan(0); - expect(response.data[0].repo).toEqual(samplesRepo); - expect(response.data[0].branch).toEqual(samplesBranch); + expect(response.data).toBeDefined(); + + const file = response.data as IGitHubFile; + expect(file.type).toBe("blob"); + validateGitHubFile(file); + expect(file.content).toBeUndefined(); }); - it("getFileContentsAsync returns a file", async () => { - const response = await gitHubClient.getFileContentsAsync( - samplesRepo.owner.login, - samplesRepo.name, - samplesBranch.name, - sampleFilePath - ); + it("getBlobAsync returns file content", async () => { + const samplesFile = SamplesContentsQueryResponse.repository.object.entries.find(file => file.type === "blob"); + const response = await gitHubClient.getBlobAsync(SamplesRepo.owner, SamplesRepo.name, samplesFile.object.oid); + expect(response.status).toBe(HttpStatusCodes.OK); - expect(response.data.path).toBe(sampleFilePath); - expect(response.data.repo).toEqual(samplesRepo); - expect(response.data.branch).toEqual(samplesBranch); + expect(response.data).toBeDefined(); + expect(typeof response.data).toBe("string"); }); }); diff --git a/src/GitHub/GitHubClient.ts b/src/GitHub/GitHubClient.ts index 281575f0e..8b3cb2b2c 100644 --- a/src/GitHub/GitHubClient.ts +++ b/src/GitHub/GitHubClient.ts @@ -1,163 +1,228 @@ import { Octokit } from "@octokit/rest"; -import { RequestHeaders } from "@octokit/types"; import { HttpStatusCodes } from "../Common/Constants"; +import { Logger } from "../Common/Logger"; +import UrlUtility from "../Common/UrlUtility"; +import { isSamplesCall, SamplesContentsQueryResponse } from "../Explorer/Notebook/NotebookSamples"; +import { NotebookUtil } from "../Explorer/Notebook/NotebookUtil"; + +export interface IGitHubPageInfo { + endCursor: string; + hasNextPage: boolean; +} export interface IGitHubResponse { status: number; data: T; + pageInfo?: IGitHubPageInfo; } export interface IGitHubRepo { - // API properties name: string; - owner: { - login: string; - }; + owner: string; private: boolean; - - // Custom properties children?: IGitHubFile[]; } export interface IGitHubFile { - // API properties - type: "file" | "dir" | "symlink" | "submodule"; - encoding?: string; - size: number; + type: "blob" | "tree"; + size?: number; name: string; path: string; content?: string; - sha: string; - - // Custom properties + sha?: string; children?: IGitHubFile[]; - repo?: IGitHubRepo; - branch?: IGitHubBranch; + repo: IGitHubRepo; + branch: IGitHubBranch; + commit: IGitHubCommit; } export interface IGitHubCommit { - // API properties sha: string; message: string; - committer: { - date: string; - }; + commitDate: string; } export interface IGitHubBranch { - // API properties name: string; } -export interface IGitHubUser { - // API properties - login: string; +// graphql schema +interface Collection { + pageInfo?: PageInfo; + nodes: T[]; +} + +interface Repository { + isPrivate: boolean; + name: string; + owner: { + login: string; + }; +} + +interface Ref { name: string; } +interface History { + history: Collection; +} + +interface Commit { + committer: { + date: string; + }; + message: string; + oid: string; +} + +interface Tree extends Blob { + entries: TreeEntry[]; +} + +interface TreeEntry { + name: string; + type: string; + object: Blob; +} + +interface Blob { + byteSize?: number; + oid?: string; +} + +interface PageInfo { + endCursor: string; + hasNextPage: boolean; +} + +// graphql queries and types +const repositoryQuery = `query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + owner { + login + } + name + isPrivate + } +}`; +type RepositoryQueryParams = { + owner: string; + repo: string; +}; +type RepositoryQueryResponse = { + repository: Repository; +}; + +const repositoriesQuery = `query($pageSize: Int!, $endCursor: String) { + viewer { + repositories(first: $pageSize, after: $endCursor) { + pageInfo { + endCursor, + hasNextPage + } + nodes { + owner { + login + } + name + isPrivate + } + } + } +}`; +type RepositoriesQueryParams = { + pageSize: number; + endCursor?: string; +}; +type RepositoriesQueryResponse = { + viewer: { + repositories: Collection; + }; +}; + +const branchesQuery = `query($owner: String!, $repo: String!, $refPrefix: String!, $pageSize: Int!, $endCursor: String) { + repository(owner: $owner, name: $repo) { + refs(refPrefix: $refPrefix, first: $pageSize, after: $endCursor) { + pageInfo { + endCursor, + hasNextPage + } + nodes { + name + } + } + } +}`; +type BranchesQueryParams = { + owner: string; + repo: string; + refPrefix: string; + pageSize: number; + endCursor?: string; +}; +type BranchesQueryResponse = { + repository: { + refs: Collection; + }; +}; + +const contentsQuery = `query($owner: String!, $repo: String!, $ref: String!, $path: String, $objectExpression: String!) { + repository(owner: $owner, name: $repo) { + owner { + login + } + name + isPrivate + ref(qualifiedName: $ref) { + name + target { + ... on Commit { + history(first: 1, path: $path) { + nodes { + oid + message + committer { + date + } + } + } + } + } + } + object(expression: $objectExpression) { + ... on Blob { + oid + byteSize + } + ... on Tree { + entries { + name + type + object { + ... on Blob { + oid + byteSize + } + } + } + } + } + } +}`; +type ContentsQueryParams = { + owner: string; + repo: string; + ref: string; + path?: string; + objectExpression: string; +}; +type ContentsQueryResponse = { + repository: Repository & { ref: Ref & { target: History } } & { object: Tree }; +}; + export class GitHubClient { - private static readonly gitHubApiEndpoint = "https://api.github.com"; - - private static readonly samplesRepo: IGitHubRepo = { - name: "cosmos-notebooks", - private: false, - owner: { - login: "Azure-Samples" - } - }; - - private static readonly samplesBranch: IGitHubBranch = { - name: "master" - }; - - private static readonly samplesTopCommit: IGitHubCommit = { - sha: "41b964f442b638097a75a3f3b6a6451db05a12bf", - committer: { - date: "2020-05-19T05:03:30Z" - }, - message: "Fixing formatting" - }; - - private static readonly samplesFiles: IGitHubFile[] = [ - { - name: ".github", - path: ".github", - sha: "5e6794a8177a0c07a8719f6e1d7b41cce6f92e1e", - size: 0, - type: "dir" - }, - { - name: ".gitignore", - path: ".gitignore", - sha: "3e759b75bf455ac809d0987d369aab89137b5689", - size: 5582, - type: "file" - }, - { - name: "1. GettingStarted.ipynb", - path: "1. GettingStarted.ipynb", - sha: "0732ff5366e4aefdc4c378c61cbd968664f0acec", - size: 3933, - type: "file" - }, - { - name: "2. Visualization.ipynb", - path: "2. Visualization.ipynb", - sha: "f480134ac4adf2f50ce5fe66836c6966749d3ca1", - size: 814261, - type: "file" - }, - { - name: "3. RequestUnits.ipynb", - path: "3. RequestUnits.ipynb", - sha: "252b79a4adc81e9f2ffde453231b695d75e270e8", - size: 9490, - type: "file" - }, - { - name: "4. Indexing.ipynb", - path: "4. Indexing.ipynb", - sha: "e10dd67bd1c55c345226769e4f80e43659ef9cd5", - size: 10394, - type: "file" - }, - { - name: "5. StoredProcedures.ipynb", - path: "5. StoredProcedures.ipynb", - sha: "949941949920de4d2d111149e2182e9657cc8134", - size: 11818, - type: "file" - }, - { - name: "6. GlobalDistribution.ipynb", - path: "6. GlobalDistribution.ipynb", - sha: "b91c31dacacbc9e35750d9054063dda4a5309f3b", - size: 11375, - type: "file" - }, - { - name: "7. IoTAnomalyDetection.ipynb", - path: "7. IoTAnomalyDetection.ipynb", - sha: "82057ae52a67721a5966e2361317f5dfbd0ee595", - size: 377939, - type: "file" - }, - { - name: "All_API_quickstarts", - path: "All_API_quickstarts", - sha: "07054293e6c8fc00771fccd0cde207f5c8053978", - size: 0, - type: "dir" - }, - { - name: "CSharp_quickstarts", - path: "CSharp_quickstarts", - sha: "10e7f5704e6b56a40cac74bc39f15b7708954f52", - size: 0, - type: "dir" - } - ]; - + private static readonly SelfErrorCode = 599; private ocktokit: Octokit; constructor(token: string, private errorCallback: (error: any) => void) { @@ -169,167 +234,136 @@ export class GitHubClient { } public async getRepoAsync(owner: string, repo: string): Promise> { - if (GitHubClient.isSamplesCall(owner, repo)) { + try { + const response = (await this.ocktokit.graphql(repositoryQuery, { + owner, + repo + } as RepositoryQueryParams)) as RepositoryQueryResponse; + return { status: HttpStatusCodes.OK, - data: GitHubClient.samplesRepo + data: GitHubClient.toGitHubRepo(response.repository) + }; + } catch (error) { + GitHubClient.log(Logger.logError, `GitHubClient.getRepoAsync failed: ${error}`); + return { + status: GitHubClient.SelfErrorCode, + data: undefined }; } - - const response = await this.ocktokit.repos.get({ - owner, - repo, - headers: GitHubClient.getDisableCacheHeaders() - }); - - let data: IGitHubRepo; - if (response.data) { - data = GitHubClient.toGitHubRepo(response.data); - } - - return { status: response.status, data }; } - public async getReposAsync(page: number, perPage: number): Promise> { - const response = await this.ocktokit.repos.listForAuthenticatedUser({ - page, - per_page: perPage, - headers: GitHubClient.getDisableCacheHeaders() - }); + public async getReposAsync(pageSize: number, endCursor?: string): Promise> { + try { + const response = (await this.ocktokit.graphql(repositoriesQuery, { + pageSize, + endCursor + } as RepositoriesQueryParams)) as RepositoriesQueryResponse; - let data: IGitHubRepo[]; - if (response.data) { - data = []; - response.data?.forEach((element: any) => data.push(GitHubClient.toGitHubRepo(element))); + return { + status: HttpStatusCodes.OK, + data: response.viewer.repositories.nodes.map(repo => GitHubClient.toGitHubRepo(repo)), + pageInfo: GitHubClient.toGitHubPageInfo(response.viewer.repositories.pageInfo) + }; + } catch (error) { + GitHubClient.log(Logger.logError, `GitHubClient.getReposAsync failed: ${error}`); + return { + status: GitHubClient.SelfErrorCode, + data: undefined + }; } - - return { status: response.status, data }; } public async getBranchesAsync( owner: string, repo: string, - page: number, - perPage: number + pageSize: number, + endCursor?: string ): Promise> { - const response = await this.ocktokit.repos.listBranches({ - owner, - repo, - page, - per_page: perPage, - headers: GitHubClient.getDisableCacheHeaders() - }); + try { + const response = (await this.ocktokit.graphql(branchesQuery, { + owner, + repo, + refPrefix: "refs/heads/", + pageSize, + endCursor + } as BranchesQueryParams)) as BranchesQueryResponse; - let data: IGitHubBranch[]; - if (response.data) { - data = []; - response.data?.forEach(element => data.push(GitHubClient.toGitHubBranch(element))); - } - - return { status: response.status, data }; - } - - public async getCommitsAsync( - owner: string, - repo: string, - branch: string, - path: string, - page: number, - perPage: number - ): Promise> { - if (GitHubClient.isSamplesCall(owner, repo, branch) && path === "" && page === 1 && perPage === 1) { return { status: HttpStatusCodes.OK, - data: [GitHubClient.samplesTopCommit] + data: response.repository.refs.nodes.map(ref => GitHubClient.toGitHubBranch(ref)), + pageInfo: GitHubClient.toGitHubPageInfo(response.repository.refs.pageInfo) + }; + } catch (error) { + GitHubClient.log(Logger.logError, `GitHubClient.getBranchesAsync failed: ${error}`); + return { + status: GitHubClient.SelfErrorCode, + data: undefined }; } - - const response = await this.ocktokit.repos.listCommits({ - owner, - repo, - sha: branch, - path, - page, - per_page: perPage, - headers: GitHubClient.getDisableCacheHeaders() - }); - - let data: IGitHubCommit[]; - if (response.data) { - data = []; - response.data?.forEach(element => - data.push(GitHubClient.toGitHubCommit({ ...element.commit, sha: element.sha })) - ); - } - - return { status: response.status, data }; - } - - public async getDirContentsAsync( - owner: string, - repo: string, - branch: string, - path: string - ): Promise> { - return (await this.getContentsAsync(owner, repo, branch, path)) as IGitHubResponse; - } - - public async getFileContentsAsync( - owner: string, - repo: string, - branch: string, - path: string - ): Promise> { - return (await this.getContentsAsync(owner, repo, branch, path)) as IGitHubResponse; } public async getContentsAsync( owner: string, repo: string, branch: string, - path: string + path?: string ): Promise> { - if (GitHubClient.isSamplesCall(owner, repo, branch) && path === "") { + try { + let response: ContentsQueryResponse; + if (isSamplesCall(owner, repo, branch) && !path) { + response = SamplesContentsQueryResponse; + } else { + response = (await this.ocktokit.graphql(contentsQuery, { + owner, + repo, + ref: `refs/heads/${branch}`, + path: path || undefined, + objectExpression: `refs/heads/${branch}:${path || ""}` + } as ContentsQueryParams)) as ContentsQueryResponse; + } + + let data: IGitHubFile | IGitHubFile[]; + const entries = response.repository.object.entries; + const gitHubRepo = GitHubClient.toGitHubRepo(response.repository); + const gitHubBranch = GitHubClient.toGitHubBranch(response.repository.ref); + const gitHubCommit = GitHubClient.toGitHubCommit(response.repository.ref.target.history.nodes[0]); + + if (Array.isArray(entries)) { + data = entries.map(entry => + GitHubClient.toGitHubFile( + entry, + (path && UrlUtility.createUri(path, entry.name)) || entry.name, + gitHubRepo, + gitHubBranch, + gitHubCommit + ) + ); + } else { + data = GitHubClient.toGitHubFile( + { + name: NotebookUtil.getName(path), + type: "blob", + object: response.repository.object + }, + path, + gitHubRepo, + gitHubBranch, + gitHubCommit + ); + } + return { status: HttpStatusCodes.OK, - data: GitHubClient.samplesFiles.map(file => - GitHubClient.toGitHubFile(file, GitHubClient.samplesRepo, GitHubClient.samplesBranch) - ) + data + }; + } catch (error) { + GitHubClient.log(Logger.logError, `GitHubClient.getContentsAsync failed: ${error}`); + return { + status: GitHubClient.SelfErrorCode, + data: undefined }; } - - const response = await this.ocktokit.repos.getContents({ - owner, - repo, - path, - ref: branch, - headers: GitHubClient.getDisableCacheHeaders() - }); - - let data: IGitHubFile | IGitHubFile[]; - if (response.data) { - const repoResponse = await this.getRepoAsync(owner, repo); - if (repoResponse.data) { - const fileRepo: IGitHubRepo = GitHubClient.toGitHubRepo(repoResponse.data); - const fileBranch: IGitHubBranch = { name: branch }; - - if (Array.isArray(response.data)) { - const contents: IGitHubFile[] = []; - response.data.forEach((element: any) => - contents.push(GitHubClient.toGitHubFile(element, fileRepo, fileBranch)) - ); - data = contents; - } else { - data = GitHubClient.toGitHubFile( - { ...response.data, type: response.data.type as "file" | "dir" | "symlink" | "submodule" }, - fileRepo, - fileBranch - ); - } - } - } - - return { status: response.status, data }; } public async createOrUpdateFileAsync( @@ -372,7 +406,9 @@ export class GitHubClient { owner, repo, ref, - headers: GitHubClient.getDisableCacheHeaders() + headers: { + "If-None-Match": "" // disable 60s cache + } }); const currentTree = await this.ocktokit.git.getTree({ @@ -380,7 +416,9 @@ export class GitHubClient { repo, tree_sha: currentRef.data.object.sha, recursive: "1", - headers: GitHubClient.getDisableCacheHeaders() + headers: { + "If-None-Match": "" // disable 60s cache + } }); // API infers tree from paths so we need to filter them out @@ -425,7 +463,7 @@ export class GitHubClient { public async deleteFileAsync(file: IGitHubFile, message: string): Promise> { const response = await this.ocktokit.repos.deleteFile({ - owner: file.repo.owner.login, + owner: file.repo.owner, repo: file.repo.name, path: file.path, message, @@ -441,10 +479,31 @@ export class GitHubClient { return { status: response.status, data }; } - private initOctokit(token: string) { + public async getBlobAsync(owner: string, repo: string, sha: string): Promise> { + const response = await this.ocktokit.git.getBlob({ + owner, + repo, + file_sha: sha, + mediaType: { + format: "raw" + }, + headers: { + "If-None-Match": "" // disable 60s cache + } + }); + + return { status: response.status, data: (response.data) }; + } + + private async initOctokit(token: string) { this.ocktokit = new Octokit({ auth: token, - baseUrl: GitHubClient.gitHubApiEndpoint + log: { + debug: () => {}, + info: (message?: any) => GitHubClient.log(Logger.logInfo, message), + warn: (message?: any) => GitHubClient.log(Logger.logWarning, message), + error: (message?: any) => GitHubClient.log(Logger.logError, message) + } }); this.ocktokit.hook.error("request", error => { @@ -453,53 +512,69 @@ export class GitHubClient { }); } - private static getDisableCacheHeaders(): RequestHeaders { + private static log(logger: (message: string, area: string) => void, message?: any) { + if (message) { + message = typeof message === "string" ? message : JSON.stringify(message); + logger(message, "GitHubClient.Octokit"); + } + } + + private static toGitHubRepo(object: Repository): IGitHubRepo { return { - "If-None-Match": "" + owner: object.owner.login, + name: object.name, + private: object.isPrivate }; } - private static toGitHubRepo(element: IGitHubRepo): IGitHubRepo { + private static toGitHubBranch(object: Ref): IGitHubBranch { return { - name: element.name, - owner: { - login: element.owner.login - }, - private: element.private + name: object.name }; } - private static toGitHubBranch(element: IGitHubBranch): IGitHubBranch { + private static toGitHubCommit(object: { + message: string; + committer: { + date: string; + }; + sha?: string; + oid?: string; + }): IGitHubCommit { return { - name: element.name + sha: object.sha || object.oid, + message: object.message, + commitDate: object.committer.date }; } - private static toGitHubCommit(element: IGitHubCommit): IGitHubCommit { + private static toGitHubPageInfo(object: PageInfo): IGitHubPageInfo { return { - sha: element.sha, - message: element.message, - committer: { - date: element.committer.date - } + endCursor: object.endCursor, + hasNextPage: object.hasNextPage }; } - private static toGitHubFile(element: IGitHubFile, repo: IGitHubRepo, branch: IGitHubBranch): IGitHubFile { + private static toGitHubFile( + entry: TreeEntry, + path: string, + repo: IGitHubRepo, + branch: IGitHubBranch, + commit: IGitHubCommit + ): IGitHubFile { + if (entry.type !== "blob" && entry.type !== "tree") { + throw new Error(`Unsupported file type: ${entry.type}`); + } + return { - type: element.type, - encoding: element.encoding, - size: element.size, - name: element.name, - path: element.path, - content: element.content, - sha: element.sha, + type: entry.type, + name: entry.name, + path, repo, - branch + branch, + commit, + size: entry.object?.byteSize, + sha: entry.object?.oid }; } - - private static isSamplesCall(owner: string, repo: string, branch?: string): boolean { - return owner === "Azure-Samples" && repo === "cosmos-notebooks" && (!branch || branch === "master"); - } } diff --git a/src/GitHub/GitHubContentProvider.test.ts b/src/GitHub/GitHubContentProvider.test.ts index 9ec789df8..ef4fedf23 100644 --- a/src/GitHub/GitHubContentProvider.test.ts +++ b/src/GitHub/GitHubContentProvider.test.ts @@ -10,27 +10,30 @@ const gitHubContentProvider = new GitHubContentProvider({ gitHubClient, promptForCommitMsg: () => Promise.resolve("commit msg") }); +const gitHubCommit: IGitHubCommit = { + sha: "sha", + message: "message", + commitDate: "date" +}; const sampleFile: IGitHubFile = { - type: "file", - encoding: "encoding", + type: "blob", size: 0, name: "name.ipynb", path: "dir/name.ipynb", - content: btoa(fixture), + content: fixture, sha: "sha", repo: { - owner: { - login: "login" - }, + owner: "owner", name: "repo", private: false }, branch: { name: "branch" - } + }, + commit: gitHubCommit }; const sampleGitHubUri = GitHubUtils.toContentUri( - sampleFile.repo.owner.login, + sampleFile.repo.owner, sampleFile.repo.name, sampleFile.branch.name, sampleFile.path @@ -43,16 +46,9 @@ const sampleNotebookModel: IContent<"notebook"> = { created: "", last_modified: "date", mimetype: "application/x-ipynb+json", - content: sampleFile.content ? JSON.parse(atob(sampleFile.content)) : null, + content: sampleFile.content ? JSON.parse(sampleFile.content) : null, format: "json" }; -const gitHubCommit: IGitHubCommit = { - sha: "sha", - message: "message", - committer: { - date: "date" - } -}; describe("GitHubContentProvider remove", () => { it("errors on invalid path", async () => { @@ -125,9 +121,6 @@ describe("GitHubContentProvider get", () => { spyOn(GitHubClient.prototype, "getContentsAsync").and.returnValue( Promise.resolve({ status: HttpStatusCodes.OK, data: sampleFile }) ); - spyOn(GitHubClient.prototype, "getCommitsAsync").and.returnValue( - Promise.resolve({ status: HttpStatusCodes.OK, data: [gitHubCommit] }) - ); const response = await gitHubContentProvider.get(null, sampleGitHubUri, {}).toPromise(); expect(response).toBeDefined(); @@ -176,9 +169,6 @@ describe("GitHubContentProvider update", () => { spyOn(GitHubClient.prototype, "renameFileAsync").and.returnValue( Promise.resolve({ status: HttpStatusCodes.OK, data: gitHubCommit }) ); - spyOn(GitHubClient.prototype, "getCommitsAsync").and.returnValue( - Promise.resolve({ status: HttpStatusCodes.OK, data: [gitHubCommit] }) - ); const response = await gitHubContentProvider.update(null, sampleGitHubUri, sampleNotebookModel).toPromise(); expect(response).toBeDefined(); @@ -215,18 +205,14 @@ describe("GitHubContentProvider create", () => { spyOn(GitHubClient.prototype, "createOrUpdateFileAsync").and.returnValue( Promise.resolve({ status: HttpStatusCodes.Created, data: gitHubCommit }) ); - spyOn(GitHubClient.prototype, "getContentsAsync").and.returnValue( - Promise.resolve({ status: HttpStatusCodes.OK, data: sampleFile }) - ); const response = await gitHubContentProvider.create(null, sampleGitHubUri, sampleNotebookModel).toPromise(); expect(response).toBeDefined(); expect(response.status).toBe(HttpStatusCodes.Created); expect(gitHubClient.createOrUpdateFileAsync).toBeCalled(); - expect(gitHubClient.getContentsAsync).toBeCalled(); expect(response.response.type).toEqual(sampleNotebookModel.type); - expect(response.response.name).toEqual(sampleNotebookModel.name); - expect(response.response.path).toEqual(sampleNotebookModel.path); + expect(response.response.name).toBeDefined(); + expect(response.response.path).toBeDefined(); expect(response.response.content).toBeUndefined(); }); }); diff --git a/src/GitHub/GitHubContentProvider.ts b/src/GitHub/GitHubContentProvider.ts index da1691958..792bfae1f 100644 --- a/src/GitHub/GitHubContentProvider.ts +++ b/src/GitHub/GitHubContentProvider.ts @@ -5,7 +5,7 @@ import { AjaxResponse } from "rxjs/ajax"; import { HttpStatusCodes } from "../Common/Constants"; import { Logger } from "../Common/Logger"; import { NotebookUtil } from "../Explorer/Notebook/NotebookUtil"; -import { GitHubClient, IGitHubFile, IGitHubResponse, IGitHubCommit } from "./GitHubClient"; +import { GitHubClient, IGitHubFile, IGitHubResponse, IGitHubCommit, IGitHubBranch } from "./GitHubClient"; import { GitHubUtils } from "../Utils/GitHubUtils"; import UrlUtility from "../Common/UrlUtility"; @@ -54,23 +54,14 @@ export class GitHubContentProvider implements IContentProvider { throw new GitHubContentProviderError("Failed to get content", content.status); } - const contentInfo = GitHubUtils.fromContentUri(uri); - const commitResponse = await this.params.gitHubClient.getCommitsAsync( - contentInfo.owner, - contentInfo.repo, - contentInfo.branch, - contentInfo.path, - 1, - 1 - ); - if (commitResponse.status !== HttpStatusCodes.OK) { - throw new GitHubContentProviderError("Failed to get commit", commitResponse.status); + if (!Array.isArray(content.data) && !content.data.content && params.content !== 0) { + const file = content.data; + file.content = ( + await this.params.gitHubClient.getBlobAsync(file.repo.owner, file.repo.name, file.sha) + ).data; } - return this.createSuccessAjaxResponse( - HttpStatusCodes.OK, - this.createContentModel(uri, content.data, commitResponse.data[0], params) - ); + return this.createSuccessAjaxResponse(HttpStatusCodes.OK, this.createContentModel(uri, content.data, params)); } catch (error) { Logger.logError(error, "GitHubContentProvider/get", error.errno); return this.createErrorAjaxResponse(error); @@ -90,26 +81,26 @@ export class GitHubContentProvider implements IContentProvider { const gitHubFile = content.data as IGitHubFile; const commitMsg = await this.validateContentAndGetCommitMsg(content, "Rename", "Rename"); const newUri = model.path; + const newPath = GitHubUtils.fromContentUri(newUri).path; const response = await this.params.gitHubClient.renameFileAsync( - gitHubFile.repo.owner.login, + gitHubFile.repo.owner, gitHubFile.repo.name, gitHubFile.branch.name, commitMsg, gitHubFile.path, - GitHubUtils.fromContentUri(newUri).path + newPath ); if (response.status !== HttpStatusCodes.OK) { throw new GitHubContentProviderError("Failed to rename", response.status); } - const updatedContentResponse = await this.getContent(model.path); - if (updatedContentResponse.status !== HttpStatusCodes.OK) { - throw new GitHubContentProviderError("Failed to get content after renaming", updatedContentResponse.status); - } + gitHubFile.commit = response.data; + gitHubFile.path = newPath; + gitHubFile.name = NotebookUtil.getName(gitHubFile.path); return this.createSuccessAjaxResponse( HttpStatusCodes.OK, - this.createContentModel(newUri, updatedContentResponse.data, response.data, { content: 0 }) + this.createContentModel(newUri, gitHubFile, { content: 0 }) ); } catch (error) { Logger.logError(error, "GitHubContentProvider/update", error.errno); @@ -169,14 +160,24 @@ export class GitHubContentProvider implements IContentProvider { } const newUri = GitHubUtils.toContentUri(contentInfo.owner, contentInfo.repo, contentInfo.branch, path); - const newContentResponse = await this.getContent(newUri); - if (newContentResponse.status !== HttpStatusCodes.OK) { - throw new GitHubContentProviderError("Failed to get content after creating", newContentResponse.status); - } + const newGitHubFile: IGitHubFile = { + type: "blob", + name: NotebookUtil.getName(newUri), + path, + repo: { + owner: contentInfo.owner, + name: contentInfo.repo, + private: undefined + }, + branch: { + name: contentInfo.branch + }, + commit: response.data + }; return this.createSuccessAjaxResponse( HttpStatusCodes.Created, - this.createContentModel(newUri, newContentResponse.data, response.data, { content: 0 }) + this.createContentModel(newUri, newGitHubFile, { content: 0 }) ); } catch (error) { Logger.logError(error, "GitHubContentProvider/create", error.errno); @@ -209,7 +210,7 @@ export class GitHubContentProvider implements IContentProvider { const gitHubFile = content.data as IGitHubFile; const response = await this.params.gitHubClient.createOrUpdateFileAsync( - gitHubFile.repo.owner.login, + gitHubFile.repo.owner, gitHubFile.repo.name, gitHubFile.branch.name, gitHubFile.path, @@ -221,14 +222,11 @@ export class GitHubContentProvider implements IContentProvider { throw new GitHubContentProviderError("Failed to update", response.status); } - const savedContentResponse = await this.getContent(uri); - if (savedContentResponse.status !== HttpStatusCodes.OK) { - throw new GitHubContentProviderError("Failed to get content after updating", savedContentResponse.status); - } + gitHubFile.commit = response.data; return this.createSuccessAjaxResponse( HttpStatusCodes.OK, - this.createContentModel(uri, savedContentResponse.data, response.data, { content: 0 }) + this.createContentModel(uri, gitHubFile, { content: 0 }) ); } catch (error) { Logger.logError(error, "GitHubContentProvider/update", error.errno); @@ -283,7 +281,7 @@ export class GitHubContentProvider implements IContentProvider { return commitMsg; } - private getContent(uri: string): Promise> { + private async getContent(uri: string): Promise> { const contentInfo = GitHubUtils.fromContentUri(uri); if (contentInfo) { const { owner, repo, branch, path } = contentInfo; @@ -296,43 +294,37 @@ export class GitHubContentProvider implements IContentProvider { private createContentModel( uri: string, content: IGitHubFile | IGitHubFile[], - commit: IGitHubCommit, params: Partial ): IContent { if (Array.isArray(content)) { - return this.createDirectoryModel(uri, content, commit); + return this.createDirectoryModel(uri, content); } - if (content.type !== "file") { - return this.createDirectoryModel(uri, undefined, commit); + if (content.type === "tree") { + return this.createDirectoryModel(uri, undefined); } if (NotebookUtil.isNotebookFile(uri)) { - return this.createNotebookModel(content, commit, params); + return this.createNotebookModel(content, params); } - return this.createFileModel(content, commit, params); + return this.createFileModel(content, params); } - private createDirectoryModel( - uri: string, - gitHubFiles: IGitHubFile[] | undefined, - commit: IGitHubCommit - ): IContent<"directory"> { + private createDirectoryModel(uri: string, gitHubFiles: IGitHubFile[] | undefined): IContent<"directory"> { return { - name: GitHubUtils.fromContentUri(uri).path, + name: NotebookUtil.getName(uri), path: uri, type: "directory", writable: true, // TODO: tamitta: we don't know this info here created: "", // TODO: tamitta: we don't know this info here - last_modified: commit.committer.date, + last_modified: "", // TODO: tamitta: we don't know this info here mimetype: undefined, content: gitHubFiles?.map( (file: IGitHubFile) => this.createContentModel( - GitHubUtils.toContentUri(file.repo.owner.login, file.repo.name, file.branch.name, file.path), + GitHubUtils.toContentUri(file.repo.owner, file.repo.name, file.branch.name, file.path), file, - commit, { content: 0 } @@ -342,17 +334,12 @@ export class GitHubContentProvider implements IContentProvider { }; } - private createNotebookModel( - gitHubFile: IGitHubFile, - commit: IGitHubCommit, - params: Partial - ): IContent<"notebook"> { - const content: Notebook = - gitHubFile.content && params.content !== 0 ? JSON.parse(atob(gitHubFile.content)) : undefined; + private createNotebookModel(gitHubFile: IGitHubFile, params: Partial): IContent<"notebook"> { + const content: Notebook = gitHubFile.content && params.content !== 0 ? JSON.parse(gitHubFile.content) : undefined; return { name: gitHubFile.name, path: GitHubUtils.toContentUri( - gitHubFile.repo.owner.login, + gitHubFile.repo.owner, gitHubFile.repo.name, gitHubFile.branch.name, gitHubFile.path @@ -360,23 +347,19 @@ export class GitHubContentProvider implements IContentProvider { type: "notebook", writable: true, // TODO: tamitta: we don't know this info here created: "", // TODO: tamitta: we don't know this info here - last_modified: commit.committer.date, + last_modified: gitHubFile.commit.commitDate, mimetype: content ? "application/x-ipynb+json" : undefined, content, format: content ? "json" : undefined }; } - private createFileModel( - gitHubFile: IGitHubFile, - commit: IGitHubCommit, - params: Partial - ): IContent<"file"> { - const content: string = gitHubFile.content && params.content !== 0 ? atob(gitHubFile.content) : undefined; + private createFileModel(gitHubFile: IGitHubFile, params: Partial): IContent<"file"> { + const content: string = gitHubFile.content && params.content !== 0 ? gitHubFile.content : undefined; return { name: gitHubFile.name, path: GitHubUtils.toContentUri( - gitHubFile.repo.owner.login, + gitHubFile.repo.owner, gitHubFile.repo.name, gitHubFile.branch.name, gitHubFile.path @@ -384,7 +367,7 @@ export class GitHubContentProvider implements IContentProvider { type: "file", writable: true, // TODO: tamitta: we don't know this info here created: "", // TODO: tamitta: we don't know this info here - last_modified: commit.committer.date, + last_modified: gitHubFile.commit.commitDate, mimetype: content ? "text/plain" : undefined, content, format: content ? "text" : undefined diff --git a/src/HostedExplorer.ts b/src/HostedExplorer.ts index ee9708add..4ce76314d 100644 --- a/src/HostedExplorer.ts +++ b/src/HostedExplorer.ts @@ -713,7 +713,7 @@ class HostedExplorer { ? storedDefaultTenantId.substring(DefaultDirectoryDropdownComponent.lastVisitedKey.length) : storedDefaultTenantId; - let defaultTenant: Tenant = tenants.find(t => t.tenantId === storedDefaultTenantId); + let defaultTenant: Tenant = _.find(tenants, t => t.tenantId === storedDefaultTenantId); if (!defaultTenant) { defaultTenant = tenants[0]; LocalStorageUtility.setEntryString( @@ -830,7 +830,7 @@ class HostedExplorer { const storedAccountId = LocalStorageUtility.getEntryString(StorageKey.DatabaseAccountId); const storedSubId = storedAccountId && storedAccountId.split("subscriptions/")[1].split("/")[0]; - let defaultSub = subscriptions.find(s => s.subscriptionId === storedSubId); + let defaultSub = _.find(subscriptions, s => s.subscriptionId === storedSubId); if (!defaultSub) { defaultSub = subscriptions[0]; } @@ -932,7 +932,7 @@ class HostedExplorer { } let storedDefaultAccountId = LocalStorageUtility.getEntryString(StorageKey.DatabaseAccountId); - let defaultAccount = accounts.find(a => a.id === storedDefaultAccountId); + let defaultAccount = _.find(accounts, a => a.id === storedDefaultAccountId); if (!defaultAccount) { defaultAccount = accounts[0]; diff --git a/src/Utils/GitHubUtils.ts b/src/Utils/GitHubUtils.ts index ad0bf81f3..fa434e29b 100644 --- a/src/Utils/GitHubUtils.ts +++ b/src/Utils/GitHubUtils.ts @@ -7,6 +7,10 @@ export class GitHubUtils { // Custom scheme for github content private static readonly ContentUriPattern = /github:\/\/([^/]*)\/([^/]*)\/([^?]*)\?ref=(.*)/; + // https://github.com///blob// + // We need to support this until we move to newer scheme for quickstarts + private static readonly LegacyContentUriPattern = /https:\/\/github.com\/([^/]*)\/([^/]*)\/blob\/([^/]*)\/([^?]*)/; + public static toRepoFullName(owner: string, repo: string): string { return `${owner}/${repo}`; } @@ -27,7 +31,7 @@ export class GitHubUtils { public static fromContentUri( contentUri: string ): undefined | { owner: string; repo: string; branch: string; path: string } { - const matches = contentUri.match(GitHubUtils.ContentUriPattern); + let matches = contentUri.match(GitHubUtils.ContentUriPattern); if (matches && matches.length > 4) { return { owner: matches[1], @@ -37,6 +41,18 @@ export class GitHubUtils { }; } + matches = contentUri.match(GitHubUtils.LegacyContentUriPattern); + if (matches && matches.length > 4) { + console.log(`Using legacy github content uri scheme ${contentUri}`); + + return { + owner: matches[1], + repo: matches[2], + branch: matches[3], + path: matches[4] + }; + } + return undefined; } diff --git a/src/Utils/JunoUtils.ts b/src/Utils/JunoUtils.ts index 1f95f3e60..289412930 100644 --- a/src/Utils/JunoUtils.ts +++ b/src/Utils/JunoUtils.ts @@ -54,7 +54,7 @@ export class JunoUtils { public static toPinnedRepo(item: RepoListItem): IPinnedRepo { return { - owner: item.repo.owner.login, + owner: item.repo.owner, name: item.repo.name, private: item.repo.private, branches: item.branches.map(element => ({ name: element.name })) @@ -63,9 +63,7 @@ export class JunoUtils { public static toGitHubRepo(pinnedRepo: IPinnedRepo): IGitHubRepo { return { - owner: { - login: pinnedRepo.owner - }, + owner: pinnedRepo.owner, name: pinnedRepo.name, private: pinnedRepo.private };