diff --git a/.eslintignore b/.eslintignore index fbda0d477..4e26bfc06 100644 --- a/.eslintignore +++ b/.eslintignore @@ -45,7 +45,6 @@ src/Definitions/jquery.d.ts src/Definitions/plotly.js-cartesian-dist.d-min.ts src/Definitions/png.d.ts src/Definitions/svg.d.ts -src/Definitions/worker.d.ts src/Explorer/ComponentRegisterer.test.ts src/Explorer/ComponentRegisterer.ts src/Explorer/ContextMenuButtonFactory.ts @@ -248,7 +247,6 @@ src/Utils/QueryUtils.test.ts src/applyExplorerBindings.ts src/global.d.ts src/setupTests.ts -src/workers/upload/index.ts src/Explorer/Controls/AccessibleElement/AccessibleElement.tsx src/Explorer/Controls/Accordion/AccordionComponent.tsx src/Explorer/Controls/AccountSwitch/AccountSwitchComponent.test.tsx diff --git a/jest.config.js b/jest.config.js index ca967e8b5..b1e027c3f 100644 --- a/jest.config.js +++ b/jest.config.js @@ -69,7 +69,6 @@ module.exports = { moduleNameMapper: { "^.*[.](svg|png|gif|less|css)$": "/mockModule", "@nteract/stateful-components/(.*)$": "/mockModule", - "worker-loader": "/mockModule", "office-ui-fabric-react/lib/(.*)$": "office-ui-fabric-react/lib-commonjs/$1", // https://github.com/OfficeDev/office-ui-fabric-react/wiki/Fabric-6-Release-Notes "^dnd-core$": "dnd-core/dist/cjs", "^react-dnd$": "react-dnd/dist/cjs", diff --git a/package-lock.json b/package-lock.json index 8c72a7eb9..74e9d525c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -117,27 +117,41 @@ } }, "@azure/cosmos": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/@azure/cosmos/-/cosmos-3.9.0.tgz", - "integrity": "sha512-SA+QB54I8Dvg/ZolHpsEDLK/sbSB9sFmSU1ElnMTFw88TVik+LYHq4o/srU2TY6Gr1BketjPmgLVEqrmnRvjkw==", + "version": "3.10.5", + "resolved": "https://registry.npmjs.org/@azure/cosmos/-/cosmos-3.10.5.tgz", + "integrity": "sha512-if1uApYNjNXzB+reNFvzEBHvinxdQOzU8fni9e9Fs9jcPv9m76t2pzmYJNrxxCiFLP0vbNr/QCfQzIPQVw6v/A==", "requires": { - "@types/debug": "^4.1.4", + "@azure/core-auth": "^1.2.0", "debug": "^4.1.1", "fast-json-stable-stringify": "^2.0.0", "jsbi": "^3.1.3", - "node-abort-controller": "^1.0.4", + "node-abort-controller": "^1.2.0", "node-fetch": "^2.6.0", - "os-name": "^3.1.0", "priorityqueuejs": "^1.0.0", "semaphore": "^1.0.5", "tslib": "^2.0.0", - "uuid": "^8.1.0" + "universal-user-agent": "^6.0.0", + "uuid": "^8.3.0" }, "dependencies": { + "@azure/core-auth": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.3.0.tgz", + "integrity": "sha512-kSDSZBL6c0CYdhb+7KuutnKGf2geeT+bCJAgccB0DD7wmNJSsQPcF7TcuoZX83B7VK4tLz/u+8sOO/CnCsYp8A==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "tslib": "^2.0.0" + } + }, "tslib": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", - "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", + "integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==" + }, + "universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" }, "uuid": { "version": "8.3.2", @@ -2264,6 +2278,17 @@ "dev": true, "requires": { "p-limit": "^2.2.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + } } }, "path-exists": { @@ -5066,11 +5091,6 @@ "@types/d3-selection": "*" } }, - "@types/debug": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", - "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" - }, "@types/dom4": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/dom4/-/dom4-2.0.1.tgz", @@ -8461,6 +8481,14 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } } } }, @@ -11178,6 +11206,16 @@ "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "requires": { "p-limit": "^2.2.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + } } }, "path-exists": { @@ -14221,6 +14259,17 @@ "dev": true, "requires": { "p-limit": "^2.2.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + } } }, "parse-json": { @@ -16456,9 +16505,9 @@ } }, "node-abort-controller": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-1.1.0.tgz", - "integrity": "sha512-dEYmUqjtbivotqjraOe8UvhT/poFfog1BQRNsZm/MSEDDESk2cQ1tvD8kGyuN07TM/zoW+n42odL8zTeJupYdQ==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-1.2.1.tgz", + "integrity": "sha512-79PYeJuj6S9+yOHirR0JBLFOgjB6sQCir10uN6xRx25iD+ZD4ULqgRn3MwWBRaQGB0vEgReJzWwJo42T1R6YbQ==" }, "node-fetch": { "version": "2.6.1", @@ -17122,20 +17171,22 @@ "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", "dev": true }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { - "p-try": "^2.0.0" - } - }, "p-locate": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "requires": { "p-limit": "^2.0.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + } } }, "p-map": { @@ -17954,6 +18005,17 @@ "dev": true, "requires": { "p-limit": "^2.2.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + } } }, "path-exists": { @@ -22412,28 +22474,6 @@ "errno": "~0.1.7" } }, - "worker-loader": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-2.0.0.tgz", - "integrity": "sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==", - "dev": true, - "requires": { - "loader-utils": "^1.0.0", - "schema-utils": "^0.4.0" - }, - "dependencies": { - "schema-utils": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", - "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", - "dev": true, - "requires": { - "ajv": "^6.1.0", - "ajv-keywords": "^3.1.0" - } - } - } - }, "wrap-ansi": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", diff --git a/package.json b/package.json index 853609c9e..6191b9430 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "dependencies": { "@azure/arm-cosmosdb": "9.1.0", - "@azure/cosmos": "3.9.0", + "@azure/cosmos": "3.10.5", "@azure/cosmos-language-service": "0.0.5", "@azure/identity": "1.2.1", "@azure/ms-rest-nodeauth": "3.0.7", @@ -178,8 +178,7 @@ "webpack": "4.43.0", "webpack-bundle-analyzer": "3.6.1", "webpack-cli": "3.3.10", - "webpack-dev-server": "3.11.0", - "worker-loader": "2.0.0" + "webpack-dev-server": "3.11.0" }, "scripts": { "start": "node --max-old-space-size=10196 node_modules/webpack-dev-server/bin/webpack-dev-server.js", diff --git a/src/Common/CosmosClient.ts b/src/Common/CosmosClient.ts index 749557b30..4e3029290 100644 --- a/src/Common/CosmosClient.ts +++ b/src/Common/CosmosClient.ts @@ -75,7 +75,10 @@ export async function getTokenFromAuthService(verb: string, resourceType: string } } +let _client: Cosmos.CosmosClient; + export function client(): Cosmos.CosmosClient { + if (_client) return _client; const options: Cosmos.CosmosClientOptions = { endpoint: endpoint() || "https://cosmos.azure.com", // CosmosClient gets upset if we pass a bad URL. This should never actually get called key: userContext.masterKey, @@ -89,5 +92,6 @@ export function client(): Cosmos.CosmosClient { if (configContext.PROXY_PATH !== undefined) { (options as any).plugins = [{ on: "request", plugin: requestPlugin }]; } - return new Cosmos.CosmosClient(options); + _client = new Cosmos.CosmosClient(options); + return _client; } diff --git a/src/Common/dataAccess/bulkCreateDocument.ts b/src/Common/dataAccess/bulkCreateDocument.ts new file mode 100644 index 000000000..0eb0d4475 --- /dev/null +++ b/src/Common/dataAccess/bulkCreateDocument.ts @@ -0,0 +1,36 @@ +import { JSONObject, OperationResponse } from "@azure/cosmos"; +import { CollectionBase } from "../../Contracts/ViewModels"; +import { logConsoleInfo, logConsoleProgress } from "../../Utils/NotificationConsoleUtils"; +import { client } from "../CosmosClient"; +import { handleError } from "../ErrorHandlingUtils"; + +export const bulkCreateDocument = async ( + collection: CollectionBase, + documents: JSONObject[] +): Promise => { + const clearMessage = logConsoleProgress( + `Executing ${documents.length} bulk operations for container ${collection.id()}` + ); + + try { + const response = await client() + .database(collection.databaseId) + .container(collection.id()) + .items.bulk( + documents.map((doc) => ({ operationType: "Create", resourceBody: doc })), + { continueOnError: true } + ); + + const successCount = response.filter((r) => r.statusCode === 201).length; + + logConsoleInfo( + `${documents.length} operations completed for container ${collection.id()}. ${successCount} operations succeeded` + ); + return response; + } catch (error) { + handleError(error, "BulkCreateDocument", `Error bulk creating items for container ${collection.id()}`); + throw error; + } finally { + clearMessage(); + } +}; diff --git a/src/Contracts/ViewModels.ts b/src/Contracts/ViewModels.ts index d839b3223..71d76624b 100644 --- a/src/Contracts/ViewModels.ts +++ b/src/Contracts/ViewModels.ts @@ -15,7 +15,6 @@ import StoredProcedure from "../Explorer/Tree/StoredProcedure"; import Trigger from "../Explorer/Tree/Trigger"; import UserDefinedFunction from "../Explorer/Tree/UserDefinedFunction"; import { SelfServeType } from "../SelfServe/SelfServeUtils"; -import { UploadDetails } from "../workers/upload/definitions"; import * as DataModels from "./DataModels"; import { SubscriptionType } from "./SubscriptionType"; @@ -23,6 +22,14 @@ export interface TokenProvider { getAuthHeader(): Promise; } +export interface UploadDetailsRecord { + fileName: string; + numSucceeded: number; + numFailed: number; + numThrottled: number; + errors: string[]; +} + export interface QueryResultsMetadata { hasMoreResults: boolean; firstItemIndex: number; @@ -174,7 +181,7 @@ export interface Collection extends CollectionBase { onDragOver(source: Collection, event: { originalEvent: DragEvent }): void; onDrop(source: Collection, event: { originalEvent: DragEvent }): void; - uploadFiles(fileList: FileList): Promise; + uploadFiles(fileList: FileList): Promise<{ data: UploadDetailsRecord[] }>; getLabel(): string; getPendingThroughputSplitNotification(): Promise; diff --git a/src/Definitions/worker.d.ts b/src/Definitions/worker.d.ts deleted file mode 100644 index 6a93b9569..000000000 --- a/src/Definitions/worker.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -declare module "worker-loader!*" { - class WebpackWorker extends Worker { - constructor(); - } - - export default WebpackWorker; -} diff --git a/src/Explorer/Panes/UploadItemsPane/index.tsx b/src/Explorer/Panes/UploadItemsPane/index.tsx index 1b2fee0ba..0ac55ec45 100644 --- a/src/Explorer/Panes/UploadItemsPane/index.tsx +++ b/src/Explorer/Panes/UploadItemsPane/index.tsx @@ -1,9 +1,9 @@ import { DetailsList, DetailsListLayoutMode, IColumn, SelectionMode } from "office-ui-fabric-react"; import React, { ChangeEvent, FunctionComponent, useState } from "react"; import { Upload } from "../../../Common/Upload"; +import { UploadDetailsRecord } from "../../../Contracts/ViewModels"; import { userContext } from "../../../UserContext"; import { logConsoleError } from "../../../Utils/NotificationConsoleUtils"; -import { UploadDetails, UploadDetailsRecord } from "../../../workers/upload/definitions"; import Explorer from "../../Explorer"; import { getErrorMessage } from "../../Tables/Utilities"; import { GenericRightPaneComponent, GenericRightPaneProps } from "../GenericRightPaneComponent"; @@ -13,12 +13,6 @@ export interface UploadItemsPaneProps { closePanel: () => void; } -interface IUploadFileData { - numSucceeded: number; - numFailed: number; - fileName: string; -} - const getTitle = (): string => { if (userContext.apiType === "Cassandra" || userContext.apiType === "Tables") { return "Upload Tables"; @@ -54,7 +48,7 @@ export const UploadItemsPane: FunctionComponent = ({ selectedCollection ?.uploadFiles(files) .then( - (uploadDetails: UploadDetails) => { + (uploadDetails) => { setUploadFileData(uploadDetails.data); setFiles(undefined); }, @@ -84,6 +78,7 @@ export const UploadItemsPane: FunctionComponent = ({ onClose: closePanel, onSubmit, }; + const columns: IColumn[] = [ { key: "fileName", @@ -105,12 +100,12 @@ export const UploadItemsPane: FunctionComponent = ({ }, ]; - const _renderItemColumn = (item: IUploadFileData, index: number, column: IColumn) => { + const _renderItemColumn = (item: UploadDetailsRecord, index: number, column: IColumn) => { switch (column.key) { case "status": - return {item.numSucceeded + " items created, " + item.numFailed + " errors"}; + return `${item.numSucceeded} created, ${item.numThrottled} throttled, ${item.numFailed} errors`; default: - return {item.fileName}; + return item.fileName; } }; diff --git a/src/Explorer/Tree/Collection.ts b/src/Explorer/Tree/Collection.ts index e3f0830ed..62670cf54 100644 --- a/src/Explorer/Tree/Collection.ts +++ b/src/Explorer/Tree/Collection.ts @@ -1,9 +1,8 @@ import { Resource, StoredProcedureDefinition, TriggerDefinition, UserDefinedFunctionDefinition } from "@azure/cosmos"; import * as ko from "knockout"; import * as _ from "underscore"; -import UploadWorker from "worker-loader!../../workers/upload"; -import { AuthType } from "../../AuthType"; import * as Constants from "../../Common/Constants"; +import { bulkCreateDocument } from "../../Common/dataAccess/bulkCreateDocument"; import { createDocument } from "../../Common/dataAccess/createDocument"; import { getCollectionUsageSizeInKB } from "../../Common/dataAccess/getCollectionDataUsageSize"; import { readCollectionOffer } from "../../Common/dataAccess/readCollectionOffer"; @@ -13,16 +12,13 @@ import { readUserDefinedFunctions } from "../../Common/dataAccess/readUserDefine import { getErrorMessage, getErrorStack } from "../../Common/ErrorHandlingUtils"; import * as Logger from "../../Common/Logger"; import { fetchPortalNotifications } from "../../Common/PortalNotifications"; -import { configContext, Platform } from "../../ConfigContext"; import * as DataModels from "../../Contracts/DataModels"; import * as ViewModels from "../../Contracts/ViewModels"; +import { UploadDetailsRecord } from "../../Contracts/ViewModels"; import { Action, ActionModifiers } from "../../Shared/Telemetry/TelemetryConstants"; import * as TelemetryProcessor from "../../Shared/Telemetry/TelemetryProcessor"; import { userContext } from "../../UserContext"; -import * as NotificationConsoleUtils from "../../Utils/NotificationConsoleUtils"; -import { StartUploadMessageParams, UploadDetails, UploadDetailsRecord } from "../../workers/upload/definitions"; import Explorer from "../Explorer"; -import { ConsoleDataType } from "../Menus/NotificationConsole/NotificationConsoleComponent"; import { CassandraAPIDataClient, CassandraTableKey, CassandraTableKeys } from "../Tables/TableDataClient"; import ConflictsTab from "../Tabs/ConflictsTab"; import DocumentsTab from "../Tabs/DocumentsTab"; @@ -958,73 +954,6 @@ export default class Collection implements ViewModels.Collection { this.uploadFiles(event.originalEvent.dataTransfer.files); } - public uploadFiles = (fileList: FileList): Promise => { - // TODO: right now web worker is not working with AAD flow. Use main thread for upload for now until we have backend upload capability - if (configContext.platform === Platform.Hosted && userContext.authType === AuthType.AAD) { - return this._uploadFilesCors(fileList); - } - const documentUploader: Worker = new UploadWorker(); - let inProgressNotificationId: string = ""; - - if (!fileList || fileList.length === 0) { - return Promise.reject("No files specified"); - } - - const onmessage = (resolve: (value: UploadDetails) => void, reject: (reason: any) => void, event: MessageEvent) => { - const numSuccessful: number = event.data.numUploadsSuccessful; - const numFailed: number = event.data.numUploadsFailed; - const runtimeError: string = event.data.runtimeError; - const uploadDetails: UploadDetails = event.data.uploadDetails; - - NotificationConsoleUtils.clearInProgressMessageWithId(inProgressNotificationId); - documentUploader.terminate(); - if (!!runtimeError) { - reject(runtimeError); - } else if (numSuccessful === 0) { - // all uploads failed - NotificationConsoleUtils.logConsoleError(`Failed to upload all documents to container ${this.id()}`); - } else if (numFailed > 0) { - NotificationConsoleUtils.logConsoleError( - `Failed to upload ${numFailed} of ${numSuccessful + numFailed} documents to container ${this.id()}` - ); - } else { - NotificationConsoleUtils.logConsoleInfo( - `Successfully uploaded all ${numSuccessful} documents to container ${this.id()}` - ); - } - this._logUploadDetailsInConsole(uploadDetails); - resolve(uploadDetails); - }; - function onerror(reject: (reason: any) => void, event: ErrorEvent) { - documentUploader.terminate(); - reject(event.error); - } - - const uploaderMessage: StartUploadMessageParams = { - files: fileList, - documentClientParams: { - databaseId: this.databaseId, - containerId: this.id(), - masterKey: userContext.masterKey, - endpoint: userContext.endpoint, - accessToken: userContext.accessToken, - platform: configContext.platform, - databaseAccount: userContext.databaseAccount, - }, - }; - - return new Promise((resolve, reject) => { - documentUploader.onmessage = onmessage.bind(null, resolve, reject); - documentUploader.onerror = onerror.bind(null, reject); - - documentUploader.postMessage(uploaderMessage); - inProgressNotificationId = NotificationConsoleUtils.logConsoleMessage( - ConsoleDataType.InProgress, - `Uploading and creating documents in container ${this.id()}` - ); - }); - }; - public async getPendingThroughputSplitNotification(): Promise { if (!this.container) { return undefined; @@ -1060,13 +989,13 @@ export default class Collection implements ViewModels.Collection { } } - private async _uploadFilesCors(files: FileList): Promise { - const data = await Promise.all(Array.from(files).map((file) => this._uploadFile(file))); + public async uploadFiles(files: FileList): Promise<{ data: UploadDetailsRecord[] }> { + const data = await Promise.all(Array.from(files).map((file) => this.uploadFile(file))); return { data }; } - private _uploadFile(file: File): Promise { + private uploadFile(file: File): Promise { const reader = new FileReader(); const onload = (resolve: (value: UploadDetailsRecord) => void, evt: any): void => { const fileData: string = evt.target.result; @@ -1077,6 +1006,7 @@ export default class Collection implements ViewModels.Collection { resolve({ fileName: file.name, numSucceeded: 0, + numThrottled: 0, numFailed: 1, errors: [(evt as any).error.message], }); @@ -1094,21 +1024,32 @@ export default class Collection implements ViewModels.Collection { fileName: fileName, numSucceeded: 0, numFailed: 0, + numThrottled: 0, errors: [], }; try { - const content = JSON.parse(documentContent); - - if (Array.isArray(content)) { - await Promise.all( - content.map(async (documentContent) => { - await createDocument(this, documentContent); - record.numSucceeded++; - }) + const parsedContent = JSON.parse(documentContent); + if (Array.isArray(parsedContent)) { + const chunkSize = 50; // 100 is the max # of bulk operations the SDK currently accepts but usually results in throttles on 400RU collections + const chunkedContent = Array.from({ length: Math.ceil(parsedContent.length / chunkSize) }, (_, index) => + parsedContent.slice(index * chunkSize, index * chunkSize + chunkSize) ); + for (const chunk of chunkedContent) { + const responses = await bulkCreateDocument(this, chunk); + for (const response of responses) { + if (response.statusCode === 201) { + record.numSucceeded++; + } else if (response.statusCode === 429) { + record.numThrottled++; + } else { + record.numFailed++; + record.errors = [...record.errors, `${response.statusCode} ${response.resourceBody}`]; + } + } + } } else { - await createDocument(this, documentContent); + await createDocument(this, parsedContent); record.numSucceeded++; } @@ -1120,40 +1061,6 @@ export default class Collection implements ViewModels.Collection { } } - private _logUploadDetailsInConsole(uploadDetails: UploadDetails): void { - const uploadDetailsRecords: UploadDetailsRecord[] = uploadDetails.data; - const numFiles: number = uploadDetailsRecords.length; - const stackTraceLimit: number = 100; - let stackTraceCount: number = 0; - let currentFileIndex = 0; - while (stackTraceCount < stackTraceLimit && currentFileIndex < numFiles) { - const errors: string[] = uploadDetailsRecords[currentFileIndex].errors; - for (let i = 0; i < errors.length; i++) { - if (stackTraceCount >= stackTraceLimit) { - break; - } - NotificationConsoleUtils.logConsoleMessage( - ConsoleDataType.Error, - `Document creation error for container ${this.id()} - file ${ - uploadDetailsRecords[currentFileIndex].fileName - }: ${errors[i]}` - ); - stackTraceCount++; - } - currentFileIndex++; - } - - uploadDetailsRecords.forEach((record: UploadDetailsRecord) => { - const consoleDataType: ConsoleDataType = record.numFailed > 0 ? ConsoleDataType.Error : ConsoleDataType.Info; - NotificationConsoleUtils.logConsoleMessage( - consoleDataType, - `Item creation summary for container ${this.id()} - file ${record.fileName}: ${ - record.numSucceeded - } items created, ${record.numFailed} errors` - ); - }); - } - /** * Top-level method that will open the correct tab type depending on account API */ diff --git a/src/workers/upload/definitions.ts b/src/workers/upload/definitions.ts deleted file mode 100644 index ff0a1941e..000000000 --- a/src/workers/upload/definitions.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { DatabaseAccount } from "../../Contracts/DataModels"; -import { Platform } from "../../ConfigContext"; - -export interface StartUploadMessageParams { - files: FileList; - documentClientParams: DocumentClientParams; -} - -export interface DocumentClientParams { - databaseId: string; - containerId: string; - masterKey: string; - endpoint: string; - accessToken: string; - platform: Platform; - databaseAccount: DatabaseAccount; -} - -export interface UploadDetailsRecord { - fileName: string; - numSucceeded: number; - numFailed: number; - errors: string[]; -} - -export interface UploadDetails { - data: UploadDetailsRecord[]; -} diff --git a/src/workers/upload/index.ts b/src/workers/upload/index.ts deleted file mode 100644 index 7cf0ecd20..000000000 --- a/src/workers/upload/index.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { DocumentClientParams, UploadDetailsRecord, UploadDetails } from "./definitions"; -import { client } from "../../Common/CosmosClient"; -import { updateConfigContext } from "../../ConfigContext"; -import { updateUserContext } from "../../UserContext"; -import { getErrorMessage } from "../../Common/ErrorHandlingUtils"; - -let numUploadsSuccessful = 0; -let numUploadsFailed = 0; -let numDocuments = 0; -let numFiles = 0; -let numFilesProcessed = 0; -let fileUploadDetails: { [key: string]: UploadDetailsRecord } = {}; -let databaseId: string; -let containerId: string; - -onerror = (event: ProgressEvent) => { - postMessage( - { - numUploadsSuccessful: numUploadsSuccessful, - numUploadsFailed: numUploadsFailed, - uploadDetails: transformDetailsMap(fileUploadDetails), - // TODO: Typescript complains about event.error below - runtimeError: (event as any).error.message, - }, - undefined - ); -}; - -onmessage = (event: MessageEvent) => { - const files: FileList = event.data.files; - const clientParams: DocumentClientParams = event.data.documentClientParams; - containerId = clientParams.containerId; - databaseId = clientParams.databaseId; - updateUserContext({ - masterKey: clientParams.masterKey, - endpoint: clientParams.endpoint, - accessToken: clientParams.accessToken, - databaseAccount: clientParams.databaseAccount, - }); - updateConfigContext({ - platform: clientParams.platform, - }); - if (!!files && files.length > 0) { - numFiles = files.length; - for (let i = 0; i < numFiles; i++) { - fileUploadDetails[files[i].name] = { - fileName: files[i].name, - numSucceeded: 0, - numFailed: 0, - errors: [], - }; - uploadFile(files[i]); - } - } else { - postMessage( - { - runtimeError: "No files specified", - }, - undefined - ); - } -}; - -function uploadFile(file: File): void { - const reader = new FileReader(); - reader.onload = (evt: any): void => { - numFilesProcessed++; - const fileData: string = evt.target.result; - createDocumentsFromFile(file.name, fileData); - }; - - reader.onerror = (evt: ProgressEvent): void => { - numFilesProcessed++; - // TODO: Typescript complains about event.error below - recordUploadDetailErrorForFile(file.name, (evt as any).error.message); - transmitResultIfUploadComplete(); - }; - - reader.readAsText(file); -} - -function createDocumentsFromFile(fileName: string, documentContent: string): void { - try { - const content = JSON.parse(documentContent); - const triggerCreateDocument: (documentContent: any) => void = (documentContent: any) => { - client() - .database(databaseId) - .container(containerId) - .items.create(documentContent) - .then((savedDoc) => { - fileUploadDetails[fileName].numSucceeded++; - numUploadsSuccessful++; - }) - .catch((error) => { - console.error(error); - recordUploadDetailErrorForFile(fileName, getErrorMessage(error)); - numUploadsFailed++; - }) - .finally(() => { - transmitResultIfUploadComplete(); - }); - }; - - if (Array.isArray(content)) { - numDocuments = numDocuments + content.length; - for (let i = 0; i < content.length; i++) { - triggerCreateDocument(content[i]); - } - } else { - numDocuments = numDocuments + 1; - triggerCreateDocument(content); - } - } catch (e) { - console.log(e); - recordUploadDetailErrorForFile(fileName, e.message); - transmitResultIfUploadComplete(); - } -} - -function transmitResultIfUploadComplete(): void { - if (numFilesProcessed === numFiles && numUploadsFailed + numUploadsSuccessful === numDocuments) { - postMessage( - { - numUploadsSuccessful: numUploadsSuccessful, - numUploadsFailed: numUploadsFailed, - uploadDetails: transformDetailsMap(fileUploadDetails), - }, - undefined - ); - } -} - -function recordUploadDetailErrorForFile(fileName: string, error: any): void { - fileUploadDetails[fileName].errors.push(error); - fileUploadDetails[fileName].numFailed++; -} - -function transformDetailsMap(map: any): UploadDetails { - let transformedResult: UploadDetailsRecord[] = []; - Object.keys(map).forEach((key: string) => { - transformedResult.push(map[key]); - }); - - return { data: transformedResult }; -} diff --git a/tsconfig.json b/tsconfig.json index 0d31cf8f3..de751b45e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,7 +14,7 @@ "target": "es2017", "experimentalDecorators": true, "emitDecoratorMetadata": true, - "lib": ["es5", "es6", "dom", "webworker.importscripts"], + "lib": ["es5", "es6", "dom"], "jsx": "react", "moduleResolution": "node", "resolveJsonModule": true, diff --git a/tsconfig.strict.json b/tsconfig.strict.json index c0972d852..481ce85d6 100644 --- a/tsconfig.strict.json +++ b/tsconfig.strict.json @@ -104,8 +104,7 @@ "./src/i18n.ts", "./src/quickstart.ts", "./src/setupTests.ts", - "./src/userContext.test.ts", - "./src/workers/upload/definitions.ts" + "./src/userContext.test.ts" ], "include": [ "src/Controls/**/*",