random-mirrors/cosmos-explorer
1
0
Fork 0
mirror of https://github.com/Azure/cosmos-explorer.git synced 2025-12-20 01:11:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement Sample data import for Fabric Home (#2101)

Browse Source 
* Implement dialog to import sample data

* Fix format

* Cosmetic fixes

* fix: update help link to point to the new documentation URL

---------

Co-authored-by: Sevo Kukol <sevoku@microsoft.com>
This commit is contained in:
Laurent Nguyen
2025-04-16 23:27:22 +02:00
committed by GitHub
parent 3470f56535
commit 6dbc412fa6
6 changed files with 311 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/Explorer/Tree/Collection.ts
View File

@@ -1,4 +1,10 @@
import { Resource, StoredProcedureDefinition, TriggerDefinition, UserDefinedFunctionDefinition } from "@azure/cosmos";
import {
JSONObject,
Resource,
StoredProcedureDefinition,
TriggerDefinition,
UserDefinedFunctionDefinition,
} from "@azure/cosmos";
import { useNotebook } from "Explorer/Notebook/useNotebook";
import { DocumentsTabV2 } from "Explorer/Tabs/DocumentsTabV2/DocumentsTabV2";
import { isFabricMirrored } from "Platform/Fabric/FabricUtil";
@@ -1086,6 +1092,56 @@ export default class Collection implements ViewModels.Collection {
});
}
public async bulkInsertDocuments(documents: JSONObject[]): Promise<{
numSucceeded: number;
numFailed: number;
numThrottled: number;
errors: string[];
}> {
const stats = {
numSucceeded: 0,
numFailed: 0,
numThrottled: 0,
errors: [] as string[],
};
const chunkSize = 100; // 100 is the max # of bulk operations the SDK currently accepts
const chunkedContent = Array.from({ length: Math.ceil(documents.length / chunkSize) }, (_, index) =>
documents.slice(index * chunkSize, index * chunkSize + chunkSize),
);
for (const chunk of chunkedContent) {
let retryAttempts = 0;
let chunkComplete = false;
let documentsToAttempt = chunk;
while (retryAttempts < 10 && !chunkComplete) {
const responses = await bulkCreateDocument(this, documentsToAttempt);
const attemptedDocuments = [...documentsToAttempt];
documentsToAttempt = [];
responses.forEach((response, index) => {
if (response.statusCode === 201) {
stats.numSucceeded++;
} else if (response.statusCode === 429) {
documentsToAttempt.push(attemptedDocuments[index]);
} else {
stats.numFailed++;
stats.errors.push(JSON.stringify(response.resourceBody));
}
});
if (documentsToAttempt.length === 0) {
chunkComplete = true;
break;
}
logConsoleInfo(
`${documentsToAttempt.length} document creations were throttled. Waiting ${retryAttempts} seconds and retrying throttled documents`,
);
retryAttempts++;
await sleep(retryAttempts);
}
}
return stats;
}
private async _createDocumentsFromFile(fileName: string, documentContent: string): Promise<UploadDetailsRecord> {
const record: UploadDetailsRecord = {
fileName: fileName,
@@ -1098,38 +1154,11 @@ export default class Collection implements ViewModels.Collection {
try {
const parsedContent = JSON.parse(documentContent);
if (Array.isArray(parsedContent)) {
const chunkSize = 100; // 100 is the max # of bulk operations the SDK currently accepts
const chunkedContent = Array.from({ length: Math.ceil(parsedContent.length / chunkSize) }, (_, index) =>
parsedContent.slice(index * chunkSize, index * chunkSize + chunkSize),
);
for (const chunk of chunkedContent) {
let retryAttempts = 0;
let chunkComplete = false;
let documentsToAttempt = chunk;
while (retryAttempts < 10 && !chunkComplete) {
const responses = await bulkCreateDocument(this, documentsToAttempt);
const attemptedDocuments = [...documentsToAttempt];
documentsToAttempt = [];
responses.forEach((response, index) => {
if (response.statusCode === 201) {
record.numSucceeded++;
} else if (response.statusCode === 429) {
documentsToAttempt.push(attemptedDocuments[index]);
} else {
record.numFailed++;
}
});
if (documentsToAttempt.length === 0) {
chunkComplete = true;
break;
}
logConsoleInfo(
`${documentsToAttempt.length} document creations were throttled. Waiting ${retryAttempts} seconds and retrying throttled documents`,
);
retryAttempts++;
await sleep(retryAttempts);
}
}
const { numSucceeded, numFailed, numThrottled, errors } = await this.bulkInsertDocuments(parsedContent);
record.numSucceeded = numSucceeded;
record.numFailed = numFailed;
record.numThrottled = numThrottled;
record.errors = errors;
} else {
await createDocument(this, parsedContent);
record.numSucceeded++;