Add analytical store schema POC (#164)

* add schema APIs to JunoClient

* start implementing buildSchemaNode

* finish getSchemaNodes

* finish implementing addSchema

* cleanup

* make schema optional

* handle undefined/null schema and fields. Also don't retry on gettting schema failures.

* fix request schema and get schema endpoints

* add feature flag

* try to get most recent schema when refreshed or initialized.

* add tests

* cleanup

* cleanup

* cleanup

* fix merge conflict typos

* fix lint errors

* fix tests and update snapshot

Co-authored-by: REDMOND\gaausfel <gaausfel@microsoft.com>

src/Explorer/Tree/Collection.ts

@@ -63,6 +63,8 @@ export default class Collection implements ViewModels.Collection {
   public throughput: ko.Computed<number>;
   public rawDataModel: DataModels.Collection;
   public analyticalStorageTtl: ko.Observable<number>;
+  public schema: DataModels.ISchema;
+  public requestSchema: () => void;
   public geospatialConfig: ko.Observable<DataModels.GeospatialConfig>;
 
   // TODO move this to API customization class
@@ -117,6 +119,8 @@ export default class Collection implements ViewModels.Collection {
     this.conflictResolutionPolicy = ko.observable(data.conflictResolutionPolicy);
     this.changeFeedPolicy = ko.observable<DataModels.ChangeFeedPolicy>(data.changeFeedPolicy);
     this.analyticalStorageTtl = ko.observable(data.analyticalStorageTtl);
+    this.schema = data.schema;
+    this.requestSchema = data.requestSchema;
     this.geospatialConfig = ko.observable(data.geospatialConfig);
 
     // TODO fix this to only replace non-excaped single quotes

src/Explorer/Tree/Database.test.ts

@@ -0,0 +1,82 @@
+import * as DataModels from "../../Contracts/DataModels";
+import * as ko from "knockout";
+import Database from "./Database";
+import Explorer from "../Explorer";
+import { HttpStatusCodes } from "../../Common/Constants";
+import { JunoClient } from "../../Juno/JunoClient";
+import { userContext, updateUserContext } from "../../UserContext";
+
+const createMockContainer = (): Explorer => {
+  const mockContainer = new Explorer();
+  return mockContainer;
+};
+
+updateUserContext({
+  subscriptionId: "fakeSubscriptionId",
+  resourceGroup: "fakeResourceGroup",
+  databaseAccount: {
+    id: "id",
+    name: "fakeName",
+    location: "fakeLocation",
+    type: "fakeType",
+    tags: undefined,
+    kind: "fakeKind",
+    properties: {
+      documentEndpoint: "fakeEndpoint",
+      tableEndpoint: "fakeEndpoint",
+      gremlinEndpoint: "fakeEndpoint",
+      cassandraEndpoint: "fakeEndpoint"
+    }
+  }
+});
+
+describe("Add Schema", () => {
+  it("should not call requestSchema or getSchema if analyticalStorageTtl is undefined", () => {
+    const collection: DataModels.Collection = {} as DataModels.Collection;
+    collection.analyticalStorageTtl = undefined;
+    const database = new Database(createMockContainer(), { id: "fakeId" });
+    database.container = createMockContainer();
+    database.container.isSchemaEnabled = ko.computed<boolean>(() => false);
+
+    database.junoClient = new JunoClient();
+    database.junoClient.requestSchema = jest.fn();
+    database.junoClient.getSchema = jest.fn();
+
+    database.addSchema(collection);
+
+    expect(database.junoClient.requestSchema).toBeCalledTimes(0);
+  });
+
+  it("should call requestSchema or getSchema if analyticalStorageTtl is not undefined", () => {
+    const collection: DataModels.Collection = { id: "fakeId" } as DataModels.Collection;
+    collection.analyticalStorageTtl = 0;
+
+    const database = new Database(createMockContainer(), {});
+    database.container = createMockContainer();
+    database.container.isSchemaEnabled = ko.computed<boolean>(() => true);
+
+    database.junoClient = new JunoClient();
+    database.junoClient.requestSchema = jest.fn();
+    database.junoClient.getSchema = jest.fn().mockResolvedValue({ status: HttpStatusCodes.OK, data: {} });
+
+    jest.useFakeTimers();
+    const interval = 5000;
+    const checkForSchema: NodeJS.Timeout = database.addSchema(collection, interval);
+    jest.advanceTimersByTime(interval + 1000);
+
+    expect(database.junoClient.requestSchema).toBeCalledWith({
+      id: undefined,
+      subscriptionId: userContext.subscriptionId,
+      resourceGroup: userContext.resourceGroup,
+      accountName: userContext.databaseAccount.name,
+      resource: `dbs/${database.id}/colls/${collection.id}`,
+      status: "new"
+    });
+    expect(checkForSchema).not.toBeNull();
+    expect(database.junoClient.getSchema).toBeCalledWith(
+      userContext.databaseAccount.name,
+      database.id(),
+      collection.id
+    );
+  });
+});

src/Explorer/Tree/Database.ts

@@ -13,6 +13,8 @@ import { ConsoleDataType } from "../Menus/NotificationConsole/NotificationConsol
 import * as Logger from "../../Common/Logger";
 import Explorer from "../Explorer";
 import { readCollections } from "../../Common/dataAccess/readCollections";
+import { JunoClient, IJunoResponse } from "../../Juno/JunoClient";
+import { userContext } from "../../UserContext";
 import { readDatabaseOffer } from "../../Common/dataAccess/readDatabaseOffer";
 import { DefaultAccountExperienceType } from "../../DefaultAccountExperienceType";
 import { fetchPortalNotifications } from "../../Common/PortalNotifications";
@@ -29,6 +31,7 @@ export default class Database implements ViewModels.Database {
   public isDatabaseExpanded: ko.Observable<boolean>;
   public isDatabaseShared: ko.Computed<boolean>;
   public selectedSubnodeKind: ko.Observable<ViewModels.CollectionTabKind>;
+  public junoClient: JunoClient;
 
   constructor(container: Explorer, data: any) {
     this.nodeKind = "Database";
@@ -43,6 +46,7 @@ export default class Database implements ViewModels.Database {
     this.isDatabaseShared = ko.pureComputed(() => {
       return this.offer && !!this.offer();
     });
+    this.junoClient = new JunoClient();
   }
 
   public onSettingsClick = () => {
@@ -184,6 +188,10 @@ export default class Database implements ViewModels.Database {
     const collections: DataModels.Collection[] = await readCollections(this.id());
     const deltaCollections = this.getDeltaCollections(collections);
 
+    collections.forEach((collection: DataModels.Collection) => {
+      this.addSchema(collection);
+    });
+
     deltaCollections.toAdd.forEach((collection: DataModels.Collection) => {
       const collectionVM: Collection = new Collection(this.container, this.id(), collection, null, null);
       collectionVMs.push(collectionVM);
@@ -308,4 +316,42 @@ export default class Database implements ViewModels.Database {
 
     this.collections(collectionsToKeep);
   }
+
+  public addSchema(collection: DataModels.Collection, interval?: number): NodeJS.Timeout {
+    let checkForSchema: NodeJS.Timeout = null;
+    interval = interval || 5000;
+
+    if (collection.analyticalStorageTtl !== undefined && this.container.isSchemaEnabled()) {
+      collection.requestSchema = () => {
+        this.junoClient.requestSchema({
+          id: undefined,
+          subscriptionId: userContext.subscriptionId,
+          resourceGroup: userContext.resourceGroup,
+          accountName: userContext.databaseAccount.name,
+          resource: `dbs/${this.id}/colls/${collection.id}`,
+          status: "new"
+        });
+        checkForSchema = setInterval(async () => {
+          const response: IJunoResponse<DataModels.ISchema> = await this.junoClient.getSchema(
+            userContext.databaseAccount.name,
+            this.id(),
+            collection.id
+          );
+
+          if (response.status >= 404) {
+            clearInterval(checkForSchema);
+          }
+
+          if (response.data !== null) {
+            clearInterval(checkForSchema);
+            collection.schema = response.data;
+          }
+        }, interval);
+      };
+
+      collection.requestSchema();
+    }
+
+    return checkForSchema;
+  }
 }

src/Explorer/Tree/ResourceTreeAdapter.test.tsx

@@ -0,0 +1,253 @@
+import * as ko from "knockout";
+import * as DataModels from "../../Contracts/DataModels";
+import * as ViewModels from "../../Contracts/ViewModels";
+import React from "react";
+import { ResourceTreeAdapter } from "./ResourceTreeAdapter";
+import { shallow } from "enzyme";
+import { TreeComponent, TreeNode, TreeComponentProps } from "../Controls/TreeComponent/TreeComponent";
+import Explorer from "../Explorer";
+import Collection from "./Collection";
+
+const schema: DataModels.ISchema = {
+  id: "fakeSchemaId",
+  accountName: "fakeAccountName",
+  resource: "dbs/FakeDbName/colls/FakeCollectionName",
+  fields: [
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "_rid",
+      path: "_rid",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 11,
+        name: "Int64"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "_ts",
+      path: "_ts",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "id",
+      path: "id",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "pk",
+      path: "pk",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "other",
+      path: "other",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "name",
+      path: "nested.name",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 11,
+        name: "Int64"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "someNumber",
+      path: "nested.someNumber",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 17,
+        name: "Double"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "anotherNumber",
+      path: "nested.anotherNumber",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    },
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "name",
+      path: "items.list.items.name",
+      maxRepetitionLevel: 1,
+      maxDefinitionLevel: 3
+    },
+    {
+      dataType: {
+        code: 11,
+        name: "Int64"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "someNumber",
+      path: "items.list.items.someNumber",
+      maxRepetitionLevel: 1,
+      maxDefinitionLevel: 3
+    },
+    {
+      dataType: {
+        code: 17,
+        name: "Double"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "anotherNumber",
+      path: "items.list.items.anotherNumber",
+      maxRepetitionLevel: 1,
+      maxDefinitionLevel: 3
+    },
+    {
+      dataType: {
+        code: 15,
+        name: "String"
+      },
+      hasNulls: true,
+      isArray: false,
+      schemaType: {
+        code: 0,
+        name: "Data"
+      },
+      name: "_etag",
+      path: "_etag",
+      maxRepetitionLevel: 0,
+      maxDefinitionLevel: 1
+    }
+  ]
+};
+
+const createMockContainer = (): Explorer => {
+  const mockContainer = new Explorer();
+  mockContainer.selectedNode = ko.observable<ViewModels.TreeNode>();
+  mockContainer.onUpdateTabsButtons = () => {
+    return;
+  };
+
+  return mockContainer;
+};
+
+const createMockCollection = (): ViewModels.Collection => {
+  const mockCollection = {} as DataModels.Collection;
+  mockCollection._rid = "fakeRid";
+  mockCollection._self = "fakeSelf";
+  mockCollection.id = "fakeId";
+  mockCollection.analyticalStorageTtl = 0;
+  mockCollection.schema = schema;
+
+  const mockCollectionVM: ViewModels.Collection = new Collection(
+    createMockContainer(),
+    "fakeDatabaseId",
+    mockCollection,
+    undefined,
+    undefined
+  );
+
+  return mockCollectionVM;
+};
+
+describe("Resource tree for schema", () => {
+  const mockContainer: Explorer = createMockContainer();
+  const resourceTree = new ResourceTreeAdapter(mockContainer);
+
+  it("should render", () => {
+    const rootNode: TreeNode = resourceTree.buildSchemaNode(createMockCollection());
+    const props: TreeComponentProps = {
+      rootNode,
+      className: "dataResourceTree"
+    };
+    const wrapper = shallow(<TreeComponent {...props} />);
+    expect(wrapper).toMatchSnapshot();
+  });
+});

src/Explorer/Tree/ResourceTreeAdapter.tsx

@@ -2,7 +2,7 @@ import * as ko from "knockout";
 import * as React from "react";
 import { ReactAdapter } from "../../Bindings/ReactBindingHandler";
 import { AccordionComponent, AccordionItemComponent } from "../Controls/Accordion/AccordionComponent";
-import { TreeComponent, TreeNode, TreeNodeMenuItem } from "../Controls/TreeComponent/TreeComponent";
+import { TreeComponent, TreeNode, TreeNodeMenuItem, TreeNodeComponent } from "../Controls/TreeComponent/TreeComponent";
 import * as ViewModels from "../../Contracts/ViewModels";
 import { NotebookContentItem, NotebookContentItemType } from "../Notebook/NotebookContentItem";
 import { ResourceTreeContextMenuButtonFactory } from "../ContextMenuButtonFactory";
@@ -32,6 +32,7 @@ import StoredProcedure from "./StoredProcedure";
 import Trigger from "./Trigger";
 import TabsBase from "../Tabs/TabsBase";
 import { userContext } from "../../UserContext";
+import * as DataModels from "../../Contracts/DataModels";
 
 export class ResourceTreeAdapter implements ReactAdapter {
   public static readonly MyNotebooksTitle = "My Notebooks";
@@ -289,6 +290,11 @@ export class ResourceTreeAdapter implements ReactAdapter {
       });
     }
 
+    const schemaNode: TreeNode = this.buildSchemaNode(collection);
+    if (schemaNode) {
+      children.push(schemaNode);
+    }
+
     if (ResourceTreeAdapter.showScriptNodes(this.container)) {
       children.push(this.buildStoredProcedureNode(collection));
       children.push(this.buildUserDefinedFunctionsNode(collection));
@@ -405,6 +411,75 @@ export class ResourceTreeAdapter implements ReactAdapter {
     };
   }
 
+  public buildSchemaNode(collection: ViewModels.Collection): TreeNode {
+    if (collection.analyticalStorageTtl() == undefined) {
+      return undefined;
+    }
+
+    if (!collection.schema || !collection.schema.fields) {
+      return undefined;
+    }
+
+    return {
+      label: "Schema",
+      children: this.getSchemaNodes(collection.schema.fields),
+      onClick: () => {
+        collection.selectedSubnodeKind(ViewModels.CollectionTabKind.Schema);
+        this.container.tabsManager.refreshActiveTab(
+          (tab: TabsBase) => tab.collection && tab.collection.rid === collection.rid
+        );
+      }
+    };
+  }
+
+  private getSchemaNodes(fields: DataModels.IDataField[]): TreeNode[] {
+    const schema: any = {};
+
+    //unflatten
+    fields.forEach((field: DataModels.IDataField, fieldIndex: number) => {
+      const path: string[] = field.path.split(".");
+      const fieldProperties = [field.dataType.name, `HasNulls: ${field.hasNulls}`];
+      let current: any = {};
+      path.forEach((name: string, pathIndex: number) => {
+        if (pathIndex === 0) {
+          if (schema[name] === undefined) {
+            if (pathIndex === path.length - 1) {
+              schema[name] = fieldProperties;
+            } else {
+              schema[name] = {};
+            }
+          }
+          current = schema[name];
+        } else {
+          if (current[name] === undefined) {
+            if (pathIndex === path.length - 1) {
+              current[name] = fieldProperties;
+            } else {
+              current[name] = {};
+            }
+          }
+          current = current[name];
+        }
+      });
+    });
+
+    const traverse = (obj: any): TreeNode[] => {
+      const children: TreeNode[] = [];
+
+      if (obj !== null && !Array.isArray(obj) && typeof obj === "object") {
+        Object.entries(obj).forEach(([key, value]) => {
+          children.push({ label: key, children: traverse(value) });
+        });
+      } else if (Array.isArray(obj)) {
+        return [{ label: obj[0] }, { label: obj[1] }];
+      }
+
+      return children;
+    };
+
+    return traverse(schema);
+  }
+
   private buildNotebooksTrees(): TreeNode {
     let notebooksTree: TreeNode = {
       label: undefined,

src/Explorer/Tree/__snapshots__/ResourceTreeAdapter.test.tsx.snap

@@ -0,0 +1,172 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Resource tree for schema should render 1`] = `
+<div
+  className="treeComponent dataResourceTree"
+>
+  <TreeNodeComponent
+    generation={0}
+    node={
+      Object {
+        "children": Array [
+          Object {
+            "children": Array [
+              Object {
+                "label": "String",
+              },
+              Object {
+                "label": "HasNulls: true",
+              },
+            ],
+            "label": "_rid",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "label": "Int64",
+              },
+              Object {
+                "label": "HasNulls: true",
+              },
+            ],
+            "label": "_ts",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "label": "String",
+              },
+              Object {
+                "label": "HasNulls: true",
+              },
+            ],
+            "label": "id",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "label": "String",
+              },
+              Object {
+                "label": "HasNulls: true",
+              },
+            ],
+            "label": "pk",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "label": "String",
+              },
+              Object {
+                "label": "HasNulls: true",
+              },
+            ],
+            "label": "other",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "children": Array [
+                  Object {
+                    "label": "String",
+                  },
+                  Object {
+                    "label": "HasNulls: true",
+                  },
+                ],
+                "label": "name",
+              },
+              Object {
+                "children": Array [
+                  Object {
+                    "label": "Int64",
+                  },
+                  Object {
+                    "label": "HasNulls: true",
+                  },
+                ],
+                "label": "someNumber",
+              },
+              Object {
+                "children": Array [
+                  Object {
+                    "label": "Double",
+                  },
+                  Object {
+                    "label": "HasNulls: true",
+                  },
+                ],
+                "label": "anotherNumber",
+              },
+            ],
+            "label": "nested",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "children": Array [
+                  Object {
+                    "children": Array [
+                      Object {
+                        "children": Array [
+                          Object {
+                            "label": "String",
+                          },
+                          Object {
+                            "label": "HasNulls: true",
+                          },
+                        ],
+                        "label": "name",
+                      },
+                      Object {
+                        "children": Array [
+                          Object {
+                            "label": "Int64",
+                          },
+                          Object {
+                            "label": "HasNulls: true",
+                          },
+                        ],
+                        "label": "someNumber",
+                      },
+                      Object {
+                        "children": Array [
+                          Object {
+                            "label": "Double",
+                          },
+                          Object {
+                            "label": "HasNulls: true",
+                          },
+                        ],
+                        "label": "anotherNumber",
+                      },
+                    ],
+                    "label": "items",
+                  },
+                ],
+                "label": "list",
+              },
+            ],
+            "label": "items",
+          },
+          Object {
+            "children": Array [
+              Object {
+                "label": "String",
+              },
+              Object {
+                "label": "HasNulls: true",
+              },
+            ],
+            "label": "_etag",
+          },
+        ],
+        "label": "Schema",
+        "onClick": [Function],
+      }
+    }
+    paddingLeft={0}
+  />
+</div>
+`;