Fix Upload Items (#682)
* Fix Upload Items * Remove debugger * Switch to bulk APIs * Address TODO Co-authored-by: Jordi Bunster <jbunster@microsoft.com>
This commit is contained in:
Steve Faulkner
GitHub
parent
f94f95e788
commit
3f8e394952
|
|
@ -45,7 +45,6 @@ src/Definitions/jquery.d.ts
|
|||
src/Definitions/plotly.js-cartesian-dist.d-min.ts
|
||||
src/Definitions/png.d.ts
|
||||
src/Definitions/svg.d.ts
|
||||
src/Definitions/worker.d.ts
|
||||
src/Explorer/ComponentRegisterer.test.ts
|
||||
src/Explorer/ComponentRegisterer.ts
|
||||
src/Explorer/ContextMenuButtonFactory.ts
|
||||
|
|
@ -248,7 +247,6 @@ src/Utils/QueryUtils.test.ts
|
|||
src/applyExplorerBindings.ts
|
||||
src/global.d.ts
|
||||
src/setupTests.ts
|
||||
src/workers/upload/index.ts
|
||||
src/Explorer/Controls/AccessibleElement/AccessibleElement.tsx
|
||||
src/Explorer/Controls/Accordion/AccordionComponent.tsx
|
||||
src/Explorer/Controls/AccountSwitch/AccountSwitchComponent.test.tsx
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ module.exports = {
|
|||
moduleNameMapper: {
|
||||
"^.*[.](svg|png|gif|less|css)$": "<rootDir>/mockModule",
|
||||
"@nteract/stateful-components/(.*)$": "<rootDir>/mockModule",
|
||||
"worker-loader": "<rootDir>/mockModule",
|
||||
"office-ui-fabric-react/lib/(.*)$": "office-ui-fabric-react/lib-commonjs/$1", // https://github.com/OfficeDev/office-ui-fabric-react/wiki/Fabric-6-Release-Notes
|
||||
"^dnd-core$": "dnd-core/dist/cjs",
|
||||
"^react-dnd$": "react-dnd/dist/cjs",
|
||||
|
|
|
|||
|
|
@ -117,27 +117,41 @@
|
|||
}
|
||||
},
|
||||
"@azure/cosmos": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@azure/cosmos/-/cosmos-3.9.0.tgz",
|
||||
"integrity": "sha512-SA+QB54I8Dvg/ZolHpsEDLK/sbSB9sFmSU1ElnMTFw88TVik+LYHq4o/srU2TY6Gr1BketjPmgLVEqrmnRvjkw==",
|
||||
"version": "3.10.5",
|
||||
"resolved": "https://registry.npmjs.org/@azure/cosmos/-/cosmos-3.10.5.tgz",
|
||||
"integrity": "sha512-if1uApYNjNXzB+reNFvzEBHvinxdQOzU8fni9e9Fs9jcPv9m76t2pzmYJNrxxCiFLP0vbNr/QCfQzIPQVw6v/A==",
|
||||
"requires": {
|
||||
"@types/debug": "^4.1.4",
|
||||
"@azure/core-auth": "^1.2.0",
|
||||
"debug": "^4.1.1",
|
||||
"fast-json-stable-stringify": "^2.0.0",
|
||||
"jsbi": "^3.1.3",
|
||||
"node-abort-controller": "^1.0.4",
|
||||
"node-abort-controller": "^1.2.0",
|
||||
"node-fetch": "^2.6.0",
|
||||
"os-name": "^3.1.0",
|
||||
"priorityqueuejs": "^1.0.0",
|
||||
"semaphore": "^1.0.5",
|
||||
"tslib": "^2.0.0",
|
||||
"uuid": "^8.1.0"
|
||||
"universal-user-agent": "^6.0.0",
|
||||
"uuid": "^8.3.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@azure/core-auth": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.3.0.tgz",
|
||||
"integrity": "sha512-kSDSZBL6c0CYdhb+7KuutnKGf2geeT+bCJAgccB0DD7wmNJSsQPcF7TcuoZX83B7VK4tLz/u+8sOO/CnCsYp8A==",
|
||||
"requires": {
|
||||
"@azure/abort-controller": "^1.0.0",
|
||||
"tslib": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"tslib": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz",
|
||||
"integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A=="
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz",
|
||||
"integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w=="
|
||||
},
|
||||
"universal-user-agent": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
|
||||
"integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w=="
|
||||
},
|
||||
"uuid": {
|
||||
"version": "8.3.2",
|
||||
|
|
@ -2264,6 +2278,17 @@
|
|||
"dev": true,
|
||||
"requires": {
|
||||
"p-limit": "^2.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"path-exists": {
|
||||
|
|
@ -5066,11 +5091,6 @@
|
|||
"@types/d3-selection": "*"
|
||||
}
|
||||
},
|
||||
"@types/debug": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz",
|
||||
"integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ=="
|
||||
},
|
||||
"@types/dom4": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/dom4/-/dom4-2.0.1.tgz",
|
||||
|
|
@ -8461,6 +8481,14 @@
|
|||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
|
||||
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
|
||||
},
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -11178,6 +11206,16 @@
|
|||
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
|
||||
"requires": {
|
||||
"p-limit": "^2.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"path-exists": {
|
||||
|
|
@ -14221,6 +14259,17 @@
|
|||
"dev": true,
|
||||
"requires": {
|
||||
"p-limit": "^2.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"parse-json": {
|
||||
|
|
@ -16456,9 +16505,9 @@
|
|||
}
|
||||
},
|
||||
"node-abort-controller": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-1.1.0.tgz",
|
||||
"integrity": "sha512-dEYmUqjtbivotqjraOe8UvhT/poFfog1BQRNsZm/MSEDDESk2cQ1tvD8kGyuN07TM/zoW+n42odL8zTeJupYdQ=="
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-1.2.1.tgz",
|
||||
"integrity": "sha512-79PYeJuj6S9+yOHirR0JBLFOgjB6sQCir10uN6xRx25iD+ZD4ULqgRn3MwWBRaQGB0vEgReJzWwJo42T1R6YbQ=="
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
|
|
@ -17122,6 +17171,14 @@
|
|||
"integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==",
|
||||
"dev": true
|
||||
},
|
||||
"p-locate": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
|
||||
"integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
|
||||
"requires": {
|
||||
"p-limit": "^2.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
|
|
@ -17129,13 +17186,7 @@
|
|||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"p-locate": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
|
||||
"integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
|
||||
"requires": {
|
||||
"p-limit": "^2.0.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"p-map": {
|
||||
|
|
@ -17954,6 +18005,17 @@
|
|||
"dev": true,
|
||||
"requires": {
|
||||
"p-limit": "^2.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"path-exists": {
|
||||
|
|
@ -22412,28 +22474,6 @@
|
|||
"errno": "~0.1.7"
|
||||
}
|
||||
},
|
||||
"worker-loader": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-2.0.0.tgz",
|
||||
"integrity": "sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"loader-utils": "^1.0.0",
|
||||
"schema-utils": "^0.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"schema-utils": {
|
||||
"version": "0.4.7",
|
||||
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz",
|
||||
"integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ajv": "^6.1.0",
|
||||
"ajv-keywords": "^3.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"wrap-ansi": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz",
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
"main": "index.js",
|
||||
"dependencies": {
|
||||
"@azure/arm-cosmosdb": "9.1.0",
|
||||
"@azure/cosmos": "3.9.0",
|
||||
"@azure/cosmos": "3.10.5",
|
||||
"@azure/cosmos-language-service": "0.0.5",
|
||||
"@azure/identity": "1.2.1",
|
||||
"@azure/ms-rest-nodeauth": "3.0.7",
|
||||
|
|
@ -178,8 +178,7 @@
|
|||
"webpack": "4.43.0",
|
||||
"webpack-bundle-analyzer": "3.6.1",
|
||||
"webpack-cli": "3.3.10",
|
||||
"webpack-dev-server": "3.11.0",
|
||||
"worker-loader": "2.0.0"
|
||||
"webpack-dev-server": "3.11.0"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node --max-old-space-size=10196 node_modules/webpack-dev-server/bin/webpack-dev-server.js",
|
||||
|
|
|
|||
|
|
@ -75,7 +75,10 @@ export async function getTokenFromAuthService(verb: string, resourceType: string
|
|||
}
|
||||
}
|
||||
|
||||
let _client: Cosmos.CosmosClient;
|
||||
|
||||
export function client(): Cosmos.CosmosClient {
|
||||
if (_client) return _client;
|
||||
const options: Cosmos.CosmosClientOptions = {
|
||||
endpoint: endpoint() || "https://cosmos.azure.com", // CosmosClient gets upset if we pass a bad URL. This should never actually get called
|
||||
key: userContext.masterKey,
|
||||
|
|
@ -89,5 +92,6 @@ export function client(): Cosmos.CosmosClient {
|
|||
if (configContext.PROXY_PATH !== undefined) {
|
||||
(options as any).plugins = [{ on: "request", plugin: requestPlugin }];
|
||||
}
|
||||
return new Cosmos.CosmosClient(options);
|
||||
_client = new Cosmos.CosmosClient(options);
|
||||
return _client;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
import { JSONObject, OperationResponse } from "@azure/cosmos";
|
||||
import { CollectionBase } from "../../Contracts/ViewModels";
|
||||
import { logConsoleInfo, logConsoleProgress } from "../../Utils/NotificationConsoleUtils";
|
||||
import { client } from "../CosmosClient";
|
||||
import { handleError } from "../ErrorHandlingUtils";
|
||||
|
||||
export const bulkCreateDocument = async (
|
||||
collection: CollectionBase,
|
||||
documents: JSONObject[]
|
||||
): Promise<OperationResponse[]> => {
|
||||
const clearMessage = logConsoleProgress(
|
||||
`Executing ${documents.length} bulk operations for container ${collection.id()}`
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await client()
|
||||
.database(collection.databaseId)
|
||||
.container(collection.id())
|
||||
.items.bulk(
|
||||
documents.map((doc) => ({ operationType: "Create", resourceBody: doc })),
|
||||
{ continueOnError: true }
|
||||
);
|
||||
|
||||
const successCount = response.filter((r) => r.statusCode === 201).length;
|
||||
|
||||
logConsoleInfo(
|
||||
`${documents.length} operations completed for container ${collection.id()}. ${successCount} operations succeeded`
|
||||
);
|
||||
return response;
|
||||
} catch (error) {
|
||||
handleError(error, "BulkCreateDocument", `Error bulk creating items for container ${collection.id()}`);
|
||||
throw error;
|
||||
} finally {
|
||||
clearMessage();
|
||||
}
|
||||
};
|
||||
|
|
@ -15,7 +15,6 @@ import StoredProcedure from "../Explorer/Tree/StoredProcedure";
|
|||
import Trigger from "../Explorer/Tree/Trigger";
|
||||
import UserDefinedFunction from "../Explorer/Tree/UserDefinedFunction";
|
||||
import { SelfServeType } from "../SelfServe/SelfServeUtils";
|
||||
import { UploadDetails } from "../workers/upload/definitions";
|
||||
import * as DataModels from "./DataModels";
|
||||
import { SubscriptionType } from "./SubscriptionType";
|
||||
|
||||
|
|
@ -23,6 +22,14 @@ export interface TokenProvider {
|
|||
getAuthHeader(): Promise<Headers>;
|
||||
}
|
||||
|
||||
export interface UploadDetailsRecord {
|
||||
fileName: string;
|
||||
numSucceeded: number;
|
||||
numFailed: number;
|
||||
numThrottled: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
export interface QueryResultsMetadata {
|
||||
hasMoreResults: boolean;
|
||||
firstItemIndex: number;
|
||||
|
|
@ -174,7 +181,7 @@ export interface Collection extends CollectionBase {
|
|||
|
||||
onDragOver(source: Collection, event: { originalEvent: DragEvent }): void;
|
||||
onDrop(source: Collection, event: { originalEvent: DragEvent }): void;
|
||||
uploadFiles(fileList: FileList): Promise<UploadDetails>;
|
||||
uploadFiles(fileList: FileList): Promise<{ data: UploadDetailsRecord[] }>;
|
||||
|
||||
getLabel(): string;
|
||||
getPendingThroughputSplitNotification(): Promise<DataModels.Notification>;
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
declare module "worker-loader!*" {
|
||||
class WebpackWorker extends Worker {
|
||||
constructor();
|
||||
}
|
||||
|
||||
export default WebpackWorker;
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import { DetailsList, DetailsListLayoutMode, IColumn, SelectionMode } from "office-ui-fabric-react";
|
||||
import React, { ChangeEvent, FunctionComponent, useState } from "react";
|
||||
import { Upload } from "../../../Common/Upload";
|
||||
import { UploadDetailsRecord } from "../../../Contracts/ViewModels";
|
||||
import { userContext } from "../../../UserContext";
|
||||
import { logConsoleError } from "../../../Utils/NotificationConsoleUtils";
|
||||
import { UploadDetails, UploadDetailsRecord } from "../../../workers/upload/definitions";
|
||||
import Explorer from "../../Explorer";
|
||||
import { getErrorMessage } from "../../Tables/Utilities";
|
||||
import { GenericRightPaneComponent, GenericRightPaneProps } from "../GenericRightPaneComponent";
|
||||
|
|
@ -13,12 +13,6 @@ export interface UploadItemsPaneProps {
|
|||
closePanel: () => void;
|
||||
}
|
||||
|
||||
interface IUploadFileData {
|
||||
numSucceeded: number;
|
||||
numFailed: number;
|
||||
fileName: string;
|
||||
}
|
||||
|
||||
const getTitle = (): string => {
|
||||
if (userContext.apiType === "Cassandra" || userContext.apiType === "Tables") {
|
||||
return "Upload Tables";
|
||||
|
|
@ -54,7 +48,7 @@ export const UploadItemsPane: FunctionComponent<UploadItemsPaneProps> = ({
|
|||
selectedCollection
|
||||
?.uploadFiles(files)
|
||||
.then(
|
||||
(uploadDetails: UploadDetails) => {
|
||||
(uploadDetails) => {
|
||||
setUploadFileData(uploadDetails.data);
|
||||
setFiles(undefined);
|
||||
},
|
||||
|
|
@ -84,6 +78,7 @@ export const UploadItemsPane: FunctionComponent<UploadItemsPaneProps> = ({
|
|||
onClose: closePanel,
|
||||
onSubmit,
|
||||
};
|
||||
|
||||
const columns: IColumn[] = [
|
||||
{
|
||||
key: "fileName",
|
||||
|
|
@ -105,12 +100,12 @@ export const UploadItemsPane: FunctionComponent<UploadItemsPaneProps> = ({
|
|||
},
|
||||
];
|
||||
|
||||
const _renderItemColumn = (item: IUploadFileData, index: number, column: IColumn) => {
|
||||
const _renderItemColumn = (item: UploadDetailsRecord, index: number, column: IColumn) => {
|
||||
switch (column.key) {
|
||||
case "status":
|
||||
return <span>{item.numSucceeded + " items created, " + item.numFailed + " errors"}</span>;
|
||||
return `${item.numSucceeded} created, ${item.numThrottled} throttled, ${item.numFailed} errors`;
|
||||
default:
|
||||
return <span>{item.fileName}</span>;
|
||||
return item.fileName;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import { Resource, StoredProcedureDefinition, TriggerDefinition, UserDefinedFunctionDefinition } from "@azure/cosmos";
|
||||
import * as ko from "knockout";
|
||||
import * as _ from "underscore";
|
||||
import UploadWorker from "worker-loader!../../workers/upload";
|
||||
import { AuthType } from "../../AuthType";
|
||||
import * as Constants from "../../Common/Constants";
|
||||
import { bulkCreateDocument } from "../../Common/dataAccess/bulkCreateDocument";
|
||||
import { createDocument } from "../../Common/dataAccess/createDocument";
|
||||
import { getCollectionUsageSizeInKB } from "../../Common/dataAccess/getCollectionDataUsageSize";
|
||||
import { readCollectionOffer } from "../../Common/dataAccess/readCollectionOffer";
|
||||
|
|
@ -13,16 +12,13 @@ import { readUserDefinedFunctions } from "../../Common/dataAccess/readUserDefine
|
|||
import { getErrorMessage, getErrorStack } from "../../Common/ErrorHandlingUtils";
|
||||
import * as Logger from "../../Common/Logger";
|
||||
import { fetchPortalNotifications } from "../../Common/PortalNotifications";
|
||||
import { configContext, Platform } from "../../ConfigContext";
|
||||
import * as DataModels from "../../Contracts/DataModels";
|
||||
import * as ViewModels from "../../Contracts/ViewModels";
|
||||
import { UploadDetailsRecord } from "../../Contracts/ViewModels";
|
||||
import { Action, ActionModifiers } from "../../Shared/Telemetry/TelemetryConstants";
|
||||
import * as TelemetryProcessor from "../../Shared/Telemetry/TelemetryProcessor";
|
||||
import { userContext } from "../../UserContext";
|
||||
import * as NotificationConsoleUtils from "../../Utils/NotificationConsoleUtils";
|
||||
import { StartUploadMessageParams, UploadDetails, UploadDetailsRecord } from "../../workers/upload/definitions";
|
||||
import Explorer from "../Explorer";
|
||||
import { ConsoleDataType } from "../Menus/NotificationConsole/NotificationConsoleComponent";
|
||||
import { CassandraAPIDataClient, CassandraTableKey, CassandraTableKeys } from "../Tables/TableDataClient";
|
||||
import ConflictsTab from "../Tabs/ConflictsTab";
|
||||
import DocumentsTab from "../Tabs/DocumentsTab";
|
||||
|
|
@ -958,73 +954,6 @@ export default class Collection implements ViewModels.Collection {
|
|||
this.uploadFiles(event.originalEvent.dataTransfer.files);
|
||||
}
|
||||
|
||||
public uploadFiles = (fileList: FileList): Promise<UploadDetails> => {
|
||||
// TODO: right now web worker is not working with AAD flow. Use main thread for upload for now until we have backend upload capability
|
||||
if (configContext.platform === Platform.Hosted && userContext.authType === AuthType.AAD) {
|
||||
return this._uploadFilesCors(fileList);
|
||||
}
|
||||
const documentUploader: Worker = new UploadWorker();
|
||||
let inProgressNotificationId: string = "";
|
||||
|
||||
if (!fileList || fileList.length === 0) {
|
||||
return Promise.reject("No files specified");
|
||||
}
|
||||
|
||||
const onmessage = (resolve: (value: UploadDetails) => void, reject: (reason: any) => void, event: MessageEvent) => {
|
||||
const numSuccessful: number = event.data.numUploadsSuccessful;
|
||||
const numFailed: number = event.data.numUploadsFailed;
|
||||
const runtimeError: string = event.data.runtimeError;
|
||||
const uploadDetails: UploadDetails = event.data.uploadDetails;
|
||||
|
||||
NotificationConsoleUtils.clearInProgressMessageWithId(inProgressNotificationId);
|
||||
documentUploader.terminate();
|
||||
if (!!runtimeError) {
|
||||
reject(runtimeError);
|
||||
} else if (numSuccessful === 0) {
|
||||
// all uploads failed
|
||||
NotificationConsoleUtils.logConsoleError(`Failed to upload all documents to container ${this.id()}`);
|
||||
} else if (numFailed > 0) {
|
||||
NotificationConsoleUtils.logConsoleError(
|
||||
`Failed to upload ${numFailed} of ${numSuccessful + numFailed} documents to container ${this.id()}`
|
||||
);
|
||||
} else {
|
||||
NotificationConsoleUtils.logConsoleInfo(
|
||||
`Successfully uploaded all ${numSuccessful} documents to container ${this.id()}`
|
||||
);
|
||||
}
|
||||
this._logUploadDetailsInConsole(uploadDetails);
|
||||
resolve(uploadDetails);
|
||||
};
|
||||
function onerror(reject: (reason: any) => void, event: ErrorEvent) {
|
||||
documentUploader.terminate();
|
||||
reject(event.error);
|
||||
}
|
||||
|
||||
const uploaderMessage: StartUploadMessageParams = {
|
||||
files: fileList,
|
||||
documentClientParams: {
|
||||
databaseId: this.databaseId,
|
||||
containerId: this.id(),
|
||||
masterKey: userContext.masterKey,
|
||||
endpoint: userContext.endpoint,
|
||||
accessToken: userContext.accessToken,
|
||||
platform: configContext.platform,
|
||||
databaseAccount: userContext.databaseAccount,
|
||||
},
|
||||
};
|
||||
|
||||
return new Promise<UploadDetails>((resolve, reject) => {
|
||||
documentUploader.onmessage = onmessage.bind(null, resolve, reject);
|
||||
documentUploader.onerror = onerror.bind(null, reject);
|
||||
|
||||
documentUploader.postMessage(uploaderMessage);
|
||||
inProgressNotificationId = NotificationConsoleUtils.logConsoleMessage(
|
||||
ConsoleDataType.InProgress,
|
||||
`Uploading and creating documents in container ${this.id()}`
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
public async getPendingThroughputSplitNotification(): Promise<DataModels.Notification> {
|
||||
if (!this.container) {
|
||||
return undefined;
|
||||
|
|
@ -1060,13 +989,13 @@ export default class Collection implements ViewModels.Collection {
|
|||
}
|
||||
}
|
||||
|
||||
private async _uploadFilesCors(files: FileList): Promise<UploadDetails> {
|
||||
const data = await Promise.all(Array.from(files).map((file) => this._uploadFile(file)));
|
||||
public async uploadFiles(files: FileList): Promise<{ data: UploadDetailsRecord[] }> {
|
||||
const data = await Promise.all(Array.from(files).map((file) => this.uploadFile(file)));
|
||||
|
||||
return { data };
|
||||
}
|
||||
|
||||
private _uploadFile(file: File): Promise<UploadDetailsRecord> {
|
||||
private uploadFile(file: File): Promise<UploadDetailsRecord> {
|
||||
const reader = new FileReader();
|
||||
const onload = (resolve: (value: UploadDetailsRecord) => void, evt: any): void => {
|
||||
const fileData: string = evt.target.result;
|
||||
|
|
@ -1077,6 +1006,7 @@ export default class Collection implements ViewModels.Collection {
|
|||
resolve({
|
||||
fileName: file.name,
|
||||
numSucceeded: 0,
|
||||
numThrottled: 0,
|
||||
numFailed: 1,
|
||||
errors: [(evt as any).error.message],
|
||||
});
|
||||
|
|
@ -1094,21 +1024,32 @@ export default class Collection implements ViewModels.Collection {
|
|||
fileName: fileName,
|
||||
numSucceeded: 0,
|
||||
numFailed: 0,
|
||||
numThrottled: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
try {
|
||||
const content = JSON.parse(documentContent);
|
||||
|
||||
if (Array.isArray(content)) {
|
||||
await Promise.all(
|
||||
content.map(async (documentContent) => {
|
||||
await createDocument(this, documentContent);
|
||||
record.numSucceeded++;
|
||||
})
|
||||
const parsedContent = JSON.parse(documentContent);
|
||||
if (Array.isArray(parsedContent)) {
|
||||
const chunkSize = 50; // 100 is the max # of bulk operations the SDK currently accepts but usually results in throttles on 400RU collections
|
||||
const chunkedContent = Array.from({ length: Math.ceil(parsedContent.length / chunkSize) }, (_, index) =>
|
||||
parsedContent.slice(index * chunkSize, index * chunkSize + chunkSize)
|
||||
);
|
||||
for (const chunk of chunkedContent) {
|
||||
const responses = await bulkCreateDocument(this, chunk);
|
||||
for (const response of responses) {
|
||||
if (response.statusCode === 201) {
|
||||
record.numSucceeded++;
|
||||
} else if (response.statusCode === 429) {
|
||||
record.numThrottled++;
|
||||
} else {
|
||||
await createDocument(this, documentContent);
|
||||
record.numFailed++;
|
||||
record.errors = [...record.errors, `${response.statusCode} ${response.resourceBody}`];
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await createDocument(this, parsedContent);
|
||||
record.numSucceeded++;
|
||||
}
|
||||
|
||||
|
|
@ -1120,40 +1061,6 @@ export default class Collection implements ViewModels.Collection {
|
|||
}
|
||||
}
|
||||
|
||||
private _logUploadDetailsInConsole(uploadDetails: UploadDetails): void {
|
||||
const uploadDetailsRecords: UploadDetailsRecord[] = uploadDetails.data;
|
||||
const numFiles: number = uploadDetailsRecords.length;
|
||||
const stackTraceLimit: number = 100;
|
||||
let stackTraceCount: number = 0;
|
||||
let currentFileIndex = 0;
|
||||
while (stackTraceCount < stackTraceLimit && currentFileIndex < numFiles) {
|
||||
const errors: string[] = uploadDetailsRecords[currentFileIndex].errors;
|
||||
for (let i = 0; i < errors.length; i++) {
|
||||
if (stackTraceCount >= stackTraceLimit) {
|
||||
break;
|
||||
}
|
||||
NotificationConsoleUtils.logConsoleMessage(
|
||||
ConsoleDataType.Error,
|
||||
`Document creation error for container ${this.id()} - file ${
|
||||
uploadDetailsRecords[currentFileIndex].fileName
|
||||
}: ${errors[i]}`
|
||||
);
|
||||
stackTraceCount++;
|
||||
}
|
||||
currentFileIndex++;
|
||||
}
|
||||
|
||||
uploadDetailsRecords.forEach((record: UploadDetailsRecord) => {
|
||||
const consoleDataType: ConsoleDataType = record.numFailed > 0 ? ConsoleDataType.Error : ConsoleDataType.Info;
|
||||
NotificationConsoleUtils.logConsoleMessage(
|
||||
consoleDataType,
|
||||
`Item creation summary for container ${this.id()} - file ${record.fileName}: ${
|
||||
record.numSucceeded
|
||||
} items created, ${record.numFailed} errors`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Top-level method that will open the correct tab type depending on account API
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
import { DatabaseAccount } from "../../Contracts/DataModels";
|
||||
import { Platform } from "../../ConfigContext";
|
||||
|
||||
export interface StartUploadMessageParams {
|
||||
files: FileList;
|
||||
documentClientParams: DocumentClientParams;
|
||||
}
|
||||
|
||||
export interface DocumentClientParams {
|
||||
databaseId: string;
|
||||
containerId: string;
|
||||
masterKey: string;
|
||||
endpoint: string;
|
||||
accessToken: string;
|
||||
platform: Platform;
|
||||
databaseAccount: DatabaseAccount;
|
||||
}
|
||||
|
||||
export interface UploadDetailsRecord {
|
||||
fileName: string;
|
||||
numSucceeded: number;
|
||||
numFailed: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
export interface UploadDetails {
|
||||
data: UploadDetailsRecord[];
|
||||
}
|
||||
|
|
@ -1,145 +0,0 @@
|
|||
import { DocumentClientParams, UploadDetailsRecord, UploadDetails } from "./definitions";
|
||||
import { client } from "../../Common/CosmosClient";
|
||||
import { updateConfigContext } from "../../ConfigContext";
|
||||
import { updateUserContext } from "../../UserContext";
|
||||
import { getErrorMessage } from "../../Common/ErrorHandlingUtils";
|
||||
|
||||
let numUploadsSuccessful = 0;
|
||||
let numUploadsFailed = 0;
|
||||
let numDocuments = 0;
|
||||
let numFiles = 0;
|
||||
let numFilesProcessed = 0;
|
||||
let fileUploadDetails: { [key: string]: UploadDetailsRecord } = {};
|
||||
let databaseId: string;
|
||||
let containerId: string;
|
||||
|
||||
onerror = (event: ProgressEvent) => {
|
||||
postMessage(
|
||||
{
|
||||
numUploadsSuccessful: numUploadsSuccessful,
|
||||
numUploadsFailed: numUploadsFailed,
|
||||
uploadDetails: transformDetailsMap(fileUploadDetails),
|
||||
// TODO: Typescript complains about event.error below
|
||||
runtimeError: (event as any).error.message,
|
||||
},
|
||||
undefined
|
||||
);
|
||||
};
|
||||
|
||||
onmessage = (event: MessageEvent) => {
|
||||
const files: FileList = event.data.files;
|
||||
const clientParams: DocumentClientParams = event.data.documentClientParams;
|
||||
containerId = clientParams.containerId;
|
||||
databaseId = clientParams.databaseId;
|
||||
updateUserContext({
|
||||
masterKey: clientParams.masterKey,
|
||||
endpoint: clientParams.endpoint,
|
||||
accessToken: clientParams.accessToken,
|
||||
databaseAccount: clientParams.databaseAccount,
|
||||
});
|
||||
updateConfigContext({
|
||||
platform: clientParams.platform,
|
||||
});
|
||||
if (!!files && files.length > 0) {
|
||||
numFiles = files.length;
|
||||
for (let i = 0; i < numFiles; i++) {
|
||||
fileUploadDetails[files[i].name] = {
|
||||
fileName: files[i].name,
|
||||
numSucceeded: 0,
|
||||
numFailed: 0,
|
||||
errors: [],
|
||||
};
|
||||
uploadFile(files[i]);
|
||||
}
|
||||
} else {
|
||||
postMessage(
|
||||
{
|
||||
runtimeError: "No files specified",
|
||||
},
|
||||
undefined
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function uploadFile(file: File): void {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (evt: any): void => {
|
||||
numFilesProcessed++;
|
||||
const fileData: string = evt.target.result;
|
||||
createDocumentsFromFile(file.name, fileData);
|
||||
};
|
||||
|
||||
reader.onerror = (evt: ProgressEvent): void => {
|
||||
numFilesProcessed++;
|
||||
// TODO: Typescript complains about event.error below
|
||||
recordUploadDetailErrorForFile(file.name, (evt as any).error.message);
|
||||
transmitResultIfUploadComplete();
|
||||
};
|
||||
|
||||
reader.readAsText(file);
|
||||
}
|
||||
|
||||
function createDocumentsFromFile(fileName: string, documentContent: string): void {
|
||||
try {
|
||||
const content = JSON.parse(documentContent);
|
||||
const triggerCreateDocument: (documentContent: any) => void = (documentContent: any) => {
|
||||
client()
|
||||
.database(databaseId)
|
||||
.container(containerId)
|
||||
.items.create(documentContent)
|
||||
.then((savedDoc) => {
|
||||
fileUploadDetails[fileName].numSucceeded++;
|
||||
numUploadsSuccessful++;
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error);
|
||||
recordUploadDetailErrorForFile(fileName, getErrorMessage(error));
|
||||
numUploadsFailed++;
|
||||
})
|
||||
.finally(() => {
|
||||
transmitResultIfUploadComplete();
|
||||
});
|
||||
};
|
||||
|
||||
if (Array.isArray(content)) {
|
||||
numDocuments = numDocuments + content.length;
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
triggerCreateDocument(content[i]);
|
||||
}
|
||||
} else {
|
||||
numDocuments = numDocuments + 1;
|
||||
triggerCreateDocument(content);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
recordUploadDetailErrorForFile(fileName, e.message);
|
||||
transmitResultIfUploadComplete();
|
||||
}
|
||||
}
|
||||
|
||||
function transmitResultIfUploadComplete(): void {
|
||||
if (numFilesProcessed === numFiles && numUploadsFailed + numUploadsSuccessful === numDocuments) {
|
||||
postMessage(
|
||||
{
|
||||
numUploadsSuccessful: numUploadsSuccessful,
|
||||
numUploadsFailed: numUploadsFailed,
|
||||
uploadDetails: transformDetailsMap(fileUploadDetails),
|
||||
},
|
||||
undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function recordUploadDetailErrorForFile(fileName: string, error: any): void {
|
||||
fileUploadDetails[fileName].errors.push(error);
|
||||
fileUploadDetails[fileName].numFailed++;
|
||||
}
|
||||
|
||||
function transformDetailsMap(map: any): UploadDetails {
|
||||
let transformedResult: UploadDetailsRecord[] = [];
|
||||
Object.keys(map).forEach((key: string) => {
|
||||
transformedResult.push(map[key]);
|
||||
});
|
||||
|
||||
return { data: transformedResult };
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
"target": "es2017",
|
||||
"experimentalDecorators": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"lib": ["es5", "es6", "dom", "webworker.importscripts"],
|
||||
"lib": ["es5", "es6", "dom"],
|
||||
"jsx": "react",
|
||||
"moduleResolution": "node",
|
||||
"resolveJsonModule": true,
|
||||
|
|
|
|||
|
|
@ -104,8 +104,7 @@
|
|||
"./src/i18n.ts",
|
||||
"./src/quickstart.ts",
|
||||
"./src/setupTests.ts",
|
||||
"./src/userContext.test.ts",
|
||||
"./src/workers/upload/definitions.ts"
|
||||
"./src/userContext.test.ts"
|
||||
],
|
||||
"include": [
|
||||
"src/Controls/**/*",
|
||||
|
|
|
|||
Loading…
Reference in New Issue