Add analytical store schema POC (#164)

* add schema APIs to JunoClient

* start implementing buildSchemaNode

* finish getSchemaNodes

* finish implementing addSchema

* cleanup

* make schema optional

* handle undefined/null schema and fields. Also don't retry on gettting schema failures.

* fix request schema and get schema endpoints

* add feature flag

* try to get most recent schema when refreshed or initialized.

* add tests

* cleanup

* cleanup

* cleanup

* fix merge conflict typos

* fix lint errors

* fix tests and update snapshot

Co-authored-by: REDMOND\gaausfel <gaausfel@microsoft.com>
This commit is contained in:
Garrett Ausfeldt 2020-11-12 13:33:37 -08:00 committed by GitHub
parent addcfedd5e
commit 4ce9dcc024
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 727 additions and 1 deletions

View File

@ -125,6 +125,7 @@ export class Features {
public static readonly enableFixedCollectionWithSharedThroughput = "enablefixedcollectionwithsharedthroughput";
public static readonly ttl90Days = "ttl90days";
public static readonly enableRightPanelV2 = "enablerightpanelv2";
public static readonly enableSchema = "enableschema";
public static readonly enableSDKoperations = "enablesdkoperations";
public static readonly showMinRUSurvey = "showminrusurvey";
}

View File

@ -88,6 +88,38 @@ export interface Resource {
id: string;
}
export interface IType {
name: string;
code: number;
}
export interface IDataField {
dataType: IType;
hasNulls: boolean;
isArray: boolean;
schemaType: IType;
name: string;
path: string;
maxRepetitionLevel: number;
maxDefinitionLevel: number;
}
export interface ISchema {
id: string;
accountName: string;
resource: string;
fields: IDataField[];
}
export interface ISchemaRequest {
id: string;
subscriptionId: string;
resourceGroup: string;
accountName: string;
resource: string;
status: string;
}
export interface Collection extends Resource {
defaultTtl?: number;
indexingPolicy?: IndexingPolicy;
@ -98,6 +130,8 @@ export interface Collection extends Resource {
changeFeedPolicy?: ChangeFeedPolicy;
analyticalStorageTtl?: number;
geospatialConfig?: GeospatialConfig;
schema?: ISchema;
requestSchema?: () => void;
}
export interface Database extends Resource {

View File

@ -116,6 +116,8 @@ export interface CollectionBase extends TreeNode {
export interface Collection extends CollectionBase {
defaultTtl: ko.Observable<number>;
analyticalStorageTtl: ko.Observable<number>;
schema?: DataModels.ISchema;
requestSchema?: () => void;
indexingPolicy: ko.Observable<DataModels.IndexingPolicy>;
uniqueKeyPolicy: DataModels.UniqueKeyPolicy;
quotaInfo: ko.Observable<DataModels.CollectionQuotaInfo>;
@ -359,6 +361,7 @@ export enum CollectionTabKind {
SparkMasterTab = 16,
Gallery = 17,
NotebookViewer = 18,
Schema = 19,
SettingsV2 = 19
}

View File

@ -971,6 +971,7 @@ exports[`SettingsComponent renders 1`] = `
"isRefreshingExplorer": [Function],
"isResourceTokenCollectionNodeSelected": [Function],
"isRightPanelV2Enabled": [Function],
"isSchemaEnabled": [Function],
"isServerlessEnabled": [Function],
"isSettingsV2Enabled": [Function],
"isSparkEnabled": [Function],
@ -2251,6 +2252,7 @@ exports[`SettingsComponent renders 1`] = `
"isRefreshingExplorer": [Function],
"isResourceTokenCollectionNodeSelected": [Function],
"isRightPanelV2Enabled": [Function],
"isSchemaEnabled": [Function],
"isServerlessEnabled": [Function],
"isSettingsV2Enabled": [Function],
"isSparkEnabled": [Function],
@ -3544,6 +3546,7 @@ exports[`SettingsComponent renders 1`] = `
"isRefreshingExplorer": [Function],
"isResourceTokenCollectionNodeSelected": [Function],
"isRightPanelV2Enabled": [Function],
"isSchemaEnabled": [Function],
"isServerlessEnabled": [Function],
"isSettingsV2Enabled": [Function],
"isSparkEnabled": [Function],
@ -4824,6 +4827,7 @@ exports[`SettingsComponent renders 1`] = `
"isRefreshingExplorer": [Function],
"isResourceTokenCollectionNodeSelected": [Function],
"isRightPanelV2Enabled": [Function],
"isSchemaEnabled": [Function],
"isServerlessEnabled": [Function],
"isSettingsV2Enabled": [Function],
"isSparkEnabled": [Function],

View File

@ -226,6 +226,7 @@ export default class Explorer {
public shareTokenCopyHelperText: ko.Observable<string>;
public shouldShowDataAccessExpiryDialog: ko.Observable<boolean>;
public shouldShowContextSwitchPrompt: ko.Observable<boolean>;
public isSchemaEnabled: ko.Computed<boolean>;
// Notebooks
public isNotebookEnabled: ko.Observable<boolean>;
@ -421,6 +422,7 @@ export default class Explorer {
this.isFeatureEnabled(Constants.Features.canExceedMaximumValue)
);
this.isSchemaEnabled = ko.computed<boolean>(() => this.isFeatureEnabled(Constants.Features.enableSchema));
this.isNotificationConsoleExpanded = ko.observable<boolean>(false);
this.databases = ko.observableArray<ViewModels.Database>();

View File

@ -63,6 +63,8 @@ export default class Collection implements ViewModels.Collection {
public throughput: ko.Computed<number>;
public rawDataModel: DataModels.Collection;
public analyticalStorageTtl: ko.Observable<number>;
public schema: DataModels.ISchema;
public requestSchema: () => void;
public geospatialConfig: ko.Observable<DataModels.GeospatialConfig>;
// TODO move this to API customization class
@ -117,6 +119,8 @@ export default class Collection implements ViewModels.Collection {
this.conflictResolutionPolicy = ko.observable(data.conflictResolutionPolicy);
this.changeFeedPolicy = ko.observable<DataModels.ChangeFeedPolicy>(data.changeFeedPolicy);
this.analyticalStorageTtl = ko.observable(data.analyticalStorageTtl);
this.schema = data.schema;
this.requestSchema = data.requestSchema;
this.geospatialConfig = ko.observable(data.geospatialConfig);
// TODO fix this to only replace non-excaped single quotes

View File

@ -0,0 +1,82 @@
import * as DataModels from "../../Contracts/DataModels";
import * as ko from "knockout";
import Database from "./Database";
import Explorer from "../Explorer";
import { HttpStatusCodes } from "../../Common/Constants";
import { JunoClient } from "../../Juno/JunoClient";
import { userContext, updateUserContext } from "../../UserContext";
const createMockContainer = (): Explorer => {
const mockContainer = new Explorer();
return mockContainer;
};
updateUserContext({
subscriptionId: "fakeSubscriptionId",
resourceGroup: "fakeResourceGroup",
databaseAccount: {
id: "id",
name: "fakeName",
location: "fakeLocation",
type: "fakeType",
tags: undefined,
kind: "fakeKind",
properties: {
documentEndpoint: "fakeEndpoint",
tableEndpoint: "fakeEndpoint",
gremlinEndpoint: "fakeEndpoint",
cassandraEndpoint: "fakeEndpoint"
}
}
});
describe("Add Schema", () => {
it("should not call requestSchema or getSchema if analyticalStorageTtl is undefined", () => {
const collection: DataModels.Collection = {} as DataModels.Collection;
collection.analyticalStorageTtl = undefined;
const database = new Database(createMockContainer(), { id: "fakeId" });
database.container = createMockContainer();
database.container.isSchemaEnabled = ko.computed<boolean>(() => false);
database.junoClient = new JunoClient();
database.junoClient.requestSchema = jest.fn();
database.junoClient.getSchema = jest.fn();
database.addSchema(collection);
expect(database.junoClient.requestSchema).toBeCalledTimes(0);
});
it("should call requestSchema or getSchema if analyticalStorageTtl is not undefined", () => {
const collection: DataModels.Collection = { id: "fakeId" } as DataModels.Collection;
collection.analyticalStorageTtl = 0;
const database = new Database(createMockContainer(), {});
database.container = createMockContainer();
database.container.isSchemaEnabled = ko.computed<boolean>(() => true);
database.junoClient = new JunoClient();
database.junoClient.requestSchema = jest.fn();
database.junoClient.getSchema = jest.fn().mockResolvedValue({ status: HttpStatusCodes.OK, data: {} });
jest.useFakeTimers();
const interval = 5000;
const checkForSchema: NodeJS.Timeout = database.addSchema(collection, interval);
jest.advanceTimersByTime(interval + 1000);
expect(database.junoClient.requestSchema).toBeCalledWith({
id: undefined,
subscriptionId: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
accountName: userContext.databaseAccount.name,
resource: `dbs/${database.id}/colls/${collection.id}`,
status: "new"
});
expect(checkForSchema).not.toBeNull();
expect(database.junoClient.getSchema).toBeCalledWith(
userContext.databaseAccount.name,
database.id(),
collection.id
);
});
});

View File

@ -13,6 +13,8 @@ import { ConsoleDataType } from "../Menus/NotificationConsole/NotificationConsol
import * as Logger from "../../Common/Logger";
import Explorer from "../Explorer";
import { readCollections } from "../../Common/dataAccess/readCollections";
import { JunoClient, IJunoResponse } from "../../Juno/JunoClient";
import { userContext } from "../../UserContext";
import { readDatabaseOffer } from "../../Common/dataAccess/readDatabaseOffer";
import { DefaultAccountExperienceType } from "../../DefaultAccountExperienceType";
import { fetchPortalNotifications } from "../../Common/PortalNotifications";
@ -29,6 +31,7 @@ export default class Database implements ViewModels.Database {
public isDatabaseExpanded: ko.Observable<boolean>;
public isDatabaseShared: ko.Computed<boolean>;
public selectedSubnodeKind: ko.Observable<ViewModels.CollectionTabKind>;
public junoClient: JunoClient;
constructor(container: Explorer, data: any) {
this.nodeKind = "Database";
@ -43,6 +46,7 @@ export default class Database implements ViewModels.Database {
this.isDatabaseShared = ko.pureComputed(() => {
return this.offer && !!this.offer();
});
this.junoClient = new JunoClient();
}
public onSettingsClick = () => {
@ -184,6 +188,10 @@ export default class Database implements ViewModels.Database {
const collections: DataModels.Collection[] = await readCollections(this.id());
const deltaCollections = this.getDeltaCollections(collections);
collections.forEach((collection: DataModels.Collection) => {
this.addSchema(collection);
});
deltaCollections.toAdd.forEach((collection: DataModels.Collection) => {
const collectionVM: Collection = new Collection(this.container, this.id(), collection, null, null);
collectionVMs.push(collectionVM);
@ -308,4 +316,42 @@ export default class Database implements ViewModels.Database {
this.collections(collectionsToKeep);
}
public addSchema(collection: DataModels.Collection, interval?: number): NodeJS.Timeout {
let checkForSchema: NodeJS.Timeout = null;
interval = interval || 5000;
if (collection.analyticalStorageTtl !== undefined && this.container.isSchemaEnabled()) {
collection.requestSchema = () => {
this.junoClient.requestSchema({
id: undefined,
subscriptionId: userContext.subscriptionId,
resourceGroup: userContext.resourceGroup,
accountName: userContext.databaseAccount.name,
resource: `dbs/${this.id}/colls/${collection.id}`,
status: "new"
});
checkForSchema = setInterval(async () => {
const response: IJunoResponse<DataModels.ISchema> = await this.junoClient.getSchema(
userContext.databaseAccount.name,
this.id(),
collection.id
);
if (response.status >= 404) {
clearInterval(checkForSchema);
}
if (response.data !== null) {
clearInterval(checkForSchema);
collection.schema = response.data;
}
}, interval);
};
collection.requestSchema();
}
return checkForSchema;
}
}

View File

@ -0,0 +1,253 @@
import * as ko from "knockout";
import * as DataModels from "../../Contracts/DataModels";
import * as ViewModels from "../../Contracts/ViewModels";
import React from "react";
import { ResourceTreeAdapter } from "./ResourceTreeAdapter";
import { shallow } from "enzyme";
import { TreeComponent, TreeNode, TreeComponentProps } from "../Controls/TreeComponent/TreeComponent";
import Explorer from "../Explorer";
import Collection from "./Collection";
const schema: DataModels.ISchema = {
id: "fakeSchemaId",
accountName: "fakeAccountName",
resource: "dbs/FakeDbName/colls/FakeCollectionName",
fields: [
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "_rid",
path: "_rid",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 11,
name: "Int64"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "_ts",
path: "_ts",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "id",
path: "id",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "pk",
path: "pk",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "other",
path: "other",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "name",
path: "nested.name",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 11,
name: "Int64"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "someNumber",
path: "nested.someNumber",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 17,
name: "Double"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "anotherNumber",
path: "nested.anotherNumber",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
},
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "name",
path: "items.list.items.name",
maxRepetitionLevel: 1,
maxDefinitionLevel: 3
},
{
dataType: {
code: 11,
name: "Int64"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "someNumber",
path: "items.list.items.someNumber",
maxRepetitionLevel: 1,
maxDefinitionLevel: 3
},
{
dataType: {
code: 17,
name: "Double"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "anotherNumber",
path: "items.list.items.anotherNumber",
maxRepetitionLevel: 1,
maxDefinitionLevel: 3
},
{
dataType: {
code: 15,
name: "String"
},
hasNulls: true,
isArray: false,
schemaType: {
code: 0,
name: "Data"
},
name: "_etag",
path: "_etag",
maxRepetitionLevel: 0,
maxDefinitionLevel: 1
}
]
};
const createMockContainer = (): Explorer => {
const mockContainer = new Explorer();
mockContainer.selectedNode = ko.observable<ViewModels.TreeNode>();
mockContainer.onUpdateTabsButtons = () => {
return;
};
return mockContainer;
};
const createMockCollection = (): ViewModels.Collection => {
const mockCollection = {} as DataModels.Collection;
mockCollection._rid = "fakeRid";
mockCollection._self = "fakeSelf";
mockCollection.id = "fakeId";
mockCollection.analyticalStorageTtl = 0;
mockCollection.schema = schema;
const mockCollectionVM: ViewModels.Collection = new Collection(
createMockContainer(),
"fakeDatabaseId",
mockCollection,
undefined,
undefined
);
return mockCollectionVM;
};
describe("Resource tree for schema", () => {
const mockContainer: Explorer = createMockContainer();
const resourceTree = new ResourceTreeAdapter(mockContainer);
it("should render", () => {
const rootNode: TreeNode = resourceTree.buildSchemaNode(createMockCollection());
const props: TreeComponentProps = {
rootNode,
className: "dataResourceTree"
};
const wrapper = shallow(<TreeComponent {...props} />);
expect(wrapper).toMatchSnapshot();
});
});

View File

@ -2,7 +2,7 @@ import * as ko from "knockout";
import * as React from "react";
import { ReactAdapter } from "../../Bindings/ReactBindingHandler";
import { AccordionComponent, AccordionItemComponent } from "../Controls/Accordion/AccordionComponent";
import { TreeComponent, TreeNode, TreeNodeMenuItem } from "../Controls/TreeComponent/TreeComponent";
import { TreeComponent, TreeNode, TreeNodeMenuItem, TreeNodeComponent } from "../Controls/TreeComponent/TreeComponent";
import * as ViewModels from "../../Contracts/ViewModels";
import { NotebookContentItem, NotebookContentItemType } from "../Notebook/NotebookContentItem";
import { ResourceTreeContextMenuButtonFactory } from "../ContextMenuButtonFactory";
@ -32,6 +32,7 @@ import StoredProcedure from "./StoredProcedure";
import Trigger from "./Trigger";
import TabsBase from "../Tabs/TabsBase";
import { userContext } from "../../UserContext";
import * as DataModels from "../../Contracts/DataModels";
export class ResourceTreeAdapter implements ReactAdapter {
public static readonly MyNotebooksTitle = "My Notebooks";
@ -289,6 +290,11 @@ export class ResourceTreeAdapter implements ReactAdapter {
});
}
const schemaNode: TreeNode = this.buildSchemaNode(collection);
if (schemaNode) {
children.push(schemaNode);
}
if (ResourceTreeAdapter.showScriptNodes(this.container)) {
children.push(this.buildStoredProcedureNode(collection));
children.push(this.buildUserDefinedFunctionsNode(collection));
@ -405,6 +411,75 @@ export class ResourceTreeAdapter implements ReactAdapter {
};
}
public buildSchemaNode(collection: ViewModels.Collection): TreeNode {
if (collection.analyticalStorageTtl() == undefined) {
return undefined;
}
if (!collection.schema || !collection.schema.fields) {
return undefined;
}
return {
label: "Schema",
children: this.getSchemaNodes(collection.schema.fields),
onClick: () => {
collection.selectedSubnodeKind(ViewModels.CollectionTabKind.Schema);
this.container.tabsManager.refreshActiveTab(
(tab: TabsBase) => tab.collection && tab.collection.rid === collection.rid
);
}
};
}
private getSchemaNodes(fields: DataModels.IDataField[]): TreeNode[] {
const schema: any = {};
//unflatten
fields.forEach((field: DataModels.IDataField, fieldIndex: number) => {
const path: string[] = field.path.split(".");
const fieldProperties = [field.dataType.name, `HasNulls: ${field.hasNulls}`];
let current: any = {};
path.forEach((name: string, pathIndex: number) => {
if (pathIndex === 0) {
if (schema[name] === undefined) {
if (pathIndex === path.length - 1) {
schema[name] = fieldProperties;
} else {
schema[name] = {};
}
}
current = schema[name];
} else {
if (current[name] === undefined) {
if (pathIndex === path.length - 1) {
current[name] = fieldProperties;
} else {
current[name] = {};
}
}
current = current[name];
}
});
});
const traverse = (obj: any): TreeNode[] => {
const children: TreeNode[] = [];
if (obj !== null && !Array.isArray(obj) && typeof obj === "object") {
Object.entries(obj).forEach(([key, value]) => {
children.push({ label: key, children: traverse(value) });
});
} else if (Array.isArray(obj)) {
return [{ label: obj[0] }, { label: obj[1] }];
}
return children;
};
return traverse(schema);
}
private buildNotebooksTrees(): TreeNode {
let notebooksTree: TreeNode = {
label: undefined,

View File

@ -0,0 +1,172 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Resource tree for schema should render 1`] = `
<div
className="treeComponent dataResourceTree"
>
<TreeNodeComponent
generation={0}
node={
Object {
"children": Array [
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "_rid",
},
Object {
"children": Array [
Object {
"label": "Int64",
},
Object {
"label": "HasNulls: true",
},
],
"label": "_ts",
},
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "id",
},
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "pk",
},
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "other",
},
Object {
"children": Array [
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "name",
},
Object {
"children": Array [
Object {
"label": "Int64",
},
Object {
"label": "HasNulls: true",
},
],
"label": "someNumber",
},
Object {
"children": Array [
Object {
"label": "Double",
},
Object {
"label": "HasNulls: true",
},
],
"label": "anotherNumber",
},
],
"label": "nested",
},
Object {
"children": Array [
Object {
"children": Array [
Object {
"children": Array [
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "name",
},
Object {
"children": Array [
Object {
"label": "Int64",
},
Object {
"label": "HasNulls: true",
},
],
"label": "someNumber",
},
Object {
"children": Array [
Object {
"label": "Double",
},
Object {
"label": "HasNulls: true",
},
],
"label": "anotherNumber",
},
],
"label": "items",
},
],
"label": "list",
},
],
"label": "items",
},
Object {
"children": Array [
Object {
"label": "String",
},
Object {
"label": "HasNulls: true",
},
],
"label": "_etag",
},
],
"label": "Schema",
"onClick": [Function],
}
}
paddingLeft={0}
/>
</div>
`;

View File

@ -7,6 +7,7 @@ import { IGitHubResponse } from "../GitHub/GitHubClient";
import { IGitHubOAuthToken } from "../GitHub/GitHubOAuthService";
import { userContext } from "../UserContext";
import { getAuthorizationHeader } from "../Utils/AuthorizationUtils";
import { number } from "prop-types";
export interface IJunoResponse<T> {
status: number;
@ -427,6 +428,51 @@ export class JunoClient {
};
}
public async requestSchema(
schemaRequest: DataModels.ISchemaRequest
): Promise<IJunoResponse<DataModels.ISchemaRequest>> {
const response = await window.fetch(`${this.getAnalyticsUrl()}/${schemaRequest.accountName}/schema/request`, {
method: "POST",
body: JSON.stringify(schemaRequest),
headers: JunoClient.getHeaders()
});
let data: DataModels.ISchemaRequest;
if (response.status === HttpStatusCodes.OK) {
data = await response.json();
}
return {
status: response.status,
data
};
}
public async getSchema(
accountName: string,
databaseName: string,
containerName: string
): Promise<IJunoResponse<DataModels.ISchema>> {
const response = await window.fetch(
`${this.getAnalyticsUrl()}/${accountName}/schema/${databaseName}/${containerName}`,
{
method: "GET",
headers: JunoClient.getHeaders()
}
);
let data: DataModels.ISchema;
if (response.status === HttpStatusCodes.OK) {
data = await response.json();
}
return {
status: response.status,
data
};
}
private async getNotebooks(input: RequestInfo, init?: RequestInit): Promise<IJunoResponse<IGalleryItem[]>> {
const response = await window.fetch(input, init);
@ -457,6 +503,10 @@ export class JunoClient {
return `${this.getNotebooksUrl()}/${this.getAccount()}`;
}
private getAnalyticsUrl(): string {
return `${configContext.JUNO_ENDPOINT}/api/analytics`;
}
private static getHeaders(): HeadersInit {
const authorizationHeader = getAuthorizationHeader();
return {