RowIterable now works. RowIterable and RowScanner now include indexes of array elements.

java/pom.xml

@@ -18,12 +18,31 @@ Licensed under the MIT License.
   <url>https://github.com/Azure/azure-sdk-for-java</url>
 
   <distributionManagement>
+    <repository>
+      <name>azure-cosmos-serialization</name>
+      <layout>default</layout>
+      <id>dev-azure-com-azure-cosmos-java-azure-cosmos-serialization</id>
+      <url>https://pkgs.dev.azure.com/azure-cosmos-java/_packaging/azure-cosmos-serialization/maven/v1</url>
+    </repository>
     <site>
       <id>azure-java-build-docs</id>
       <url>${site.url}/site/${project.artifactId}</url>
     </site>
   </distributionManagement>
 
+  <repositories>
+    <repository>
+      <id>dev-azure-com-azure-cosmos-java-azure-cosmos-serialization</id>
+      <url>https://pkgs.dev.azure.com/azure-cosmos-java/_packaging/azure-cosmos-serialization/maven/v1</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
   <scm>
     <url>https://github.com/Azure/azure-sdk-for-java</url>
   </scm>

java/src/main/java/com/azure/data/cosmos/core/Out.java

@@ -59,7 +59,7 @@ public final class Out<T> {
             return true;
         }
 
-        if (!(other instanceof Out)) {
+        if (other.getClass() != Out.class) {
             return false;
         }
 

java/src/main/java/com/azure/data/cosmos/core/Reference.java

@@ -9,8 +9,8 @@ import java.util.Objects;
  * A container object which may or may not contain a non-null value.
  *
  * This is a value-based class and as such use of identity-sensitive operations--including reference equality
- * ({@code ==}), identity hash code, or synchronization--on instances of {@Reference} may have unpredictable results
+ * ({@code ==}), identity hash code, or synchronization--on instances of {@link Reference} may have unpredictable
- * and should be avoided.
+ * results and should be avoided.
  *
  * @param <T>
  */
@@ -62,7 +62,7 @@ public final class Reference<T> {
             return true;
         }
 
-        if (!(other instanceof Reference)) {
+        if (other.getClass() != Reference.class) {
             return false;
         }
 

java/src/main/java/com/azure/data/cosmos/core/Utf8String.java

@@ -465,6 +465,9 @@ public final class Utf8String implements ByteBufHolder, CharSequence, Comparable
         if (this.buffer == null) {
             return "null";
         }
+        if (this.buffer.writerIndex() == 0) {
+            return "\"\"";
+        }
         return Json.toString(this.buffer.getCharSequence(0, this.buffer.writerIndex(), UTF_8));
     }
 

java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/DataItem.java

@@ -21,9 +21,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
  */
 public class DataItem {
 
-    private final Supplier<String> fullPath;
-    private final Supplier<String> name;
-
     @JsonProperty
     private final List<String> nodes;
 
@@ -33,6 +30,9 @@ public class DataItem {
     @JsonProperty
     private final Object value;
 
+    private final Supplier<String> name;
+    private final Supplier<String> path;
+
     @SuppressWarnings("UnstableApiUsage")
     DataItem(
         @Nonnull final List<Utf8String> nodes,
@@ -56,7 +56,7 @@ public class DataItem {
 
         this.name = Suppliers.memoize(() -> this.nodes.get(this.nodes.size() - 1));
 
-        this.fullPath = Suppliers.memoize(() -> {
+        this.path = Suppliers.memoize(() -> {
 
             if (this.nodes.size() == 1) {
                 return this.nodes.get(0);
@@ -69,8 +69,11 @@ public class DataItem {
 
             int i;
 
-            for (i = 0; i < this.nodes.size() - 1; i++) {
+            for (i = 0; i < this.nodes.size() - 1; ++i) {
                 builder.append(this.nodes.get(i));
+                if (this.nodes.get(i + 1).charAt(0) != '[') {
+                    builder.append('.');
+                }
             }
 
             return builder.append(this.nodes.get(i)).toString();
@@ -86,7 +89,7 @@ public class DataItem {
     }
 
     public String path() {
-        return this.fullPath.get();
+        return this.path.get();
     }
 
     @Override

java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowIterable.java

@@ -0,0 +1,293 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.data.cosmos.serialization.hybridrow.io;
+
+import com.azure.data.cosmos.core.Out;
+import com.azure.data.cosmos.core.Utf8String;
+import com.azure.data.cosmos.serialization.hybridrow.HybridRowVersion;
+import com.azure.data.cosmos.serialization.hybridrow.Result;
+import com.azure.data.cosmos.serialization.hybridrow.RowBuffer;
+import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutResolver;
+import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutResolverNamespace;
+import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutType;
+import com.azure.data.cosmos.serialization.hybridrow.schemas.Namespace;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+import javax.annotation.Nonnull;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Stack;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.lenientFormat;
+import static java.util.Objects.requireNonNull;
+
+public class RowIterable implements AutoCloseable, Iterable<DataItem> {
+
+    private AtomicBoolean closed;
+    private final ByteBuf data;
+    private final LayoutResolver resolver;
+
+    private RowIterable(LayoutResolver resolver, ByteBuf data) {
+        this.closed = new AtomicBoolean();
+        this.data = data;
+        this.resolver = resolver;
+    }
+
+    @Override
+    public void close() {
+        if (this.closed.compareAndSet(false, true)) {
+            this.data.release();
+        }
+    }
+
+    @Nonnull
+    @Override
+    public Iterator<DataItem> iterator() {
+
+        checkState(!this.closed.get(), "RowIterable is closed");
+
+        final RowBuffer buffer = new RowBuffer(this.data, HybridRowVersion.V1, this.resolver);
+        final RowReader reader = new RowReader(buffer);
+
+        return new RowIterator(reader);
+    }
+
+    public static RowIterable open(@Nonnull Namespace namespace, @Nonnull File file) throws IOException {
+
+        checkNotNull(file, "expected non-null file");
+
+        final long length = file.length();
+        checkArgument(0 < length, "file does not exist: %s", file);
+        checkArgument(length <= Integer.MAX_VALUE, "expected file length <= %s, not %s", Integer.MAX_VALUE, length);
+
+        ByteBuf data = Unpooled.buffer((int) length);
+
+        try (InputStream stream = Files.newInputStream(file.toPath())) {
+            data.writeBytes(stream, (int) length);
+        }
+
+        LayoutResolverNamespace resolver = new LayoutResolverNamespace(namespace);
+        return new RowIterable(resolver, data);
+    }
+
+    public static RowIterable open(@Nonnull Namespace namespace, @Nonnull Path path) throws IOException {
+        return RowIterable.open(namespace, requireNonNull(path, "expected non-null path").toFile());
+    }
+
+    public static RowIterable open(@Nonnull Namespace namespace, @Nonnull String path) throws IOException {
+        return RowIterable.open(namespace, new File(requireNonNull(path, "expected non-null path")));
+    }
+
+    private static class RowIterator implements Iterator<DataItem> {
+
+        final Stack<Utf8String> paths;
+        final Stack<RowReader> readers;
+        final Out value;
+
+        DataItem dataItem;
+        RowReader reader;
+
+        RowIterator(RowReader reader) {
+            this.readers = new Stack<>();
+            this.paths = new Stack<>();
+            this.reader = reader;
+            this.value = new Out();
+        }
+
+        @Override
+        public boolean hasNext() {
+
+            while (this.dataItem == null) {
+                if (this.reader == null) {
+                    return false;
+                }
+                this.advance();
+            }
+            return true;
+        }
+
+        /**
+         * Returns the next element in the iteration.
+         *
+         * @return the next element in the iteration
+         * @throws NoSuchElementException if the iteration has no more elements
+         */
+        @Override
+        public DataItem next() {
+
+            while (this.dataItem == null) {
+                if (this.reader == null) {
+                    throw new NoSuchElementException();
+                }
+                this.advance();
+            }
+
+            DataItem dataItem = this.dataItem;
+            this.dataItem = null;
+
+            return dataItem;
+        }
+
+        @SuppressWarnings("unchecked")
+        private void advance() {
+
+            do {
+                while (this.reader.read()) {
+
+                    final Result result;
+
+                    Utf8String path = this.reader.path();
+                    checkState(!path.isNull(), "expected non-null value for path");
+
+                    LayoutType type = this.reader.type();
+                    checkState(type != null, "expected non-null type");
+
+                    switch (type.layoutCode()) {
+
+                        case BOOLEAN: {
+                            result = this.reader.readBoolean(this.value);
+                            break;
+                        }
+                        case INT_16: {
+                            result = this.reader.readInt16(this.value);
+                            break;
+                        }
+                        case INT_32: {
+                            result = this.reader.readInt32(this.value);
+                            break;
+                        }
+                        case INT_64: {
+                            result = this.reader.readInt64(this.value);
+                            break;
+                        }
+                        case UINT_8: {
+                            result = this.reader.readUInt8(this.value);
+                            break;
+                        }
+                        case UINT_32: {
+                            result = this.reader.readUInt32(this.value);
+                            break;
+                        }
+                        case UINT_64: {
+                            result = this.reader.readUInt64(this.value);
+                            break;
+                        }
+                        case BINARY: {
+                            result = this.reader.readBinary(this.value);
+                            break;
+                        }
+                        case GUID: {
+                            result = this.reader.readGuid(this.value);
+                            break;
+                        }
+                        case NULL:
+                        case BOOLEAN_FALSE:
+                        case INT_8:
+                        case UINT_16:
+                        case VAR_INT:
+                        case VAR_UINT:
+                        case FLOAT_32:
+                        case FLOAT_64:
+                        case FLOAT_128:
+                        case DECIMAL:
+                        case DATE_TIME:
+                        case UNIX_DATE_TIME:
+                        case UTF_8: {
+                            result = Result.SUCCESS;
+                            break;
+                        }
+                        case NULLABLE_SCOPE:
+                        case IMMUTABLE_NULLABLE_SCOPE: {
+                            if (!this.reader.hasValue()) {
+                                result = Result.SUCCESS;
+                                break;
+                            }
+                        }
+                        case ARRAY_SCOPE:
+                        case IMMUTABLE_ARRAY_SCOPE:
+
+                        case MAP_SCOPE:
+                        case IMMUTABLE_MAP_SCOPE:
+
+                        case OBJECT_SCOPE:
+                        case IMMUTABLE_OBJECT_SCOPE:
+
+                        case SCHEMA:
+                        case IMMUTABLE_SCHEMA:
+
+                        case SET_SCOPE:
+                        case IMMUTABLE_SET_SCOPE:
+
+                        case TAGGED2_SCOPE:
+                        case IMMUTABLE_TAGGED2_SCOPE:
+
+                        case TAGGED_SCOPE:
+                        case IMMUTABLE_TAGGED_SCOPE:
+
+                        case TUPLE_SCOPE:
+                        case IMMUTABLE_TUPLE_SCOPE:
+
+                        case TYPED_ARRAY_SCOPE:
+                        case IMMUTABLE_TYPED_ARRAY_SCOPE:
+
+                        case TYPED_MAP_SCOPE:
+                        case IMMUTABLE_TYPED_MAP_SCOPE:
+
+                        case TYPED_SET_SCOPE:
+                        case IMMUTABLE_TYPED_SET_SCOPE:
+
+                        case TYPED_TUPLE_SCOPE:
+                        case IMMUTABLE_TYPED_TUPLE_SCOPE: {
+
+                            this.readers.push(this.reader);
+
+                            this.paths.push(path.isEmpty()
+                                ? Utf8String.transcodeUtf16(lenientFormat("[%s]", this.reader.index()))
+                                : path);
+
+                            this.reader = this.reader.readScope();
+                            continue;
+                        }
+                        case MONGODB_OBJECT_ID: {
+                            throw new IllegalStateException(lenientFormat("unsupported layout type: %s", type));
+                        }
+                        case END_SCOPE:
+                        case INVALID: {
+                            throw new IllegalStateException(lenientFormat("unexpected layout type: %s", type));
+                        }
+                        default: {
+                            throw new IllegalStateException(lenientFormat("unknown layout type: %s", type));
+                        }
+                    }
+
+                    if (result != Result.SUCCESS) {
+                        String message = lenientFormat("failed to read %s value for %s", type.layoutCode(), path);
+                        throw new IllegalStateException(message);
+                    }
+
+                    this.dataItem = new DataItem(this.paths, path, type.layoutCode(), this.value.get());
+                    return;
+                }
+
+                if (this.readers.empty()) {
+                    this.reader = null;
+                } else {
+                    this.reader = this.readers.pop();
+                    this.paths.pop();
+                }
+            }
+            while (this.reader != null);
+        }
+    }
+}

java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowScanner.java

@@ -199,7 +199,10 @@ public class RowScanner implements AutoCloseable {
                 case TYPED_TUPLE_SCOPE:
                 case IMMUTABLE_TYPED_TUPLE_SCOPE: {
 
-                    visitor.nodes().push(path);
+                    visitor.nodes().push(path.isEmpty()
+                        ? Utf8String.transcodeUtf16(lenientFormat("[%s]", reader.index()))
+                        : path);
+
                     result = reader.readScope(visitor, RowScanner::visit);
                     visitor.nodes().pop();
 

java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/io/RowReaderTest.java

@@ -20,16 +20,12 @@ import org.testng.annotations.Factory;
 import org.testng.annotations.Test;
 import org.testng.util.Strings;
 
-import javax.annotation.Nonnull;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.Stack;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -45,6 +41,8 @@ import static org.testng.AssertJUnit.assertNotNull;
 
 public class RowReaderTest {
 
+    // region Fields
+
     private static final String basedir = System.getProperty("project.basedir", System.getProperty("user.dir"));
 
     private final Path dataFile;
@@ -52,6 +50,10 @@ public class RowReaderTest {
 
     private Namespace namespace;
 
+    // endregion
+
+    // region Construction and Setup
+
     private RowReaderTest(File schemaFile, Path dataFile) {
         this.schemaFile = schemaFile;
         this.dataFile = dataFile;
@@ -65,13 +67,15 @@ public class RowReaderTest {
         );
     }
 
+    // endregion
+
     @Test(groups = "unit")
     public void testIterable() throws IOException {
-
+        try (final RowIterable iterable = RowIterable.open(this.namespace, this.dataFile)) {
-        final RowIterable iterable = RowIterable.of(this.namespace, this.dataFile);
+            for (DataItem item : iterable) {
-
+                assertNotNull(item);
-        for (DataItem item : iterable) {
+                out.println(item);
-            assertNotNull(item.toString());
+            }
         }
     }
 
@@ -105,18 +109,19 @@ public class RowReaderTest {
     }
 
     @Test(groups = "unit")
-    public void testScanner() throws IOException {
+    public void testScanner() throws Exception {
-
+        try (final RowScanner scanner = RowScanner.open(this.namespace, this.dataFile)) {
-        final RowScanner scanner = RowScanner.open(this.namespace, this.dataFile);
+            scanner.visit((DataItem item, Object context) -> {
-
-        scanner.visit((DataItem item, Object context) -> {
                 assertNull(context);
                 assertNotNull(item);
                 out.println(item);
                 return Result.SUCCESS;
             }, null);
+        }
     }
 
+    // region Privates
+
     @SuppressWarnings("unchecked")
     private static Result visitFields(RowReader reader, int level) {
 
@@ -281,224 +286,5 @@ public class RowReaderTest {
         }
     }
 
-    public static class RowIterable implements Iterable<DataItem> {
+    // endregion
-
-        final ByteBuf data;
-        final LayoutResolver resolver;
-
-        private RowIterable(LayoutResolver resolver, ByteBuf data) {
-            this.resolver = resolver;
-            this.data = data;
-        }
-
-        @Nonnull
-        @Override
-        public Iterator<DataItem> iterator() {
-            final RowBuffer buffer = new RowBuffer(this.data, HybridRowVersion.V1, this.resolver);
-            final RowReader reader = new RowReader(buffer);
-            return new RowIterator(reader);
-        }
-
-        public static RowIterable of(@Nonnull Namespace namespace, @Nonnull File file) throws IOException {
-
-            checkNotNull(file, "expected non-null file");
-
-            final long length = file.length();
-            checkArgument(0 < length, "file does not exist: %s", file);
-            checkArgument(length <= Integer.MAX_VALUE, "expected file length <= %s, not %s", Integer.MAX_VALUE, length);
-
-            ByteBuf data = Unpooled.buffer((int) length);
-
-            try (InputStream stream = Files.newInputStream(file.toPath())) {
-                data.writeBytes(stream, (int) length);
-            }
-
-            LayoutResolverNamespace resolver = new LayoutResolverNamespace(namespace);
-            return new RowIterable(resolver, data);
-        }
-
-        public static RowIterable of(@Nonnull Namespace namespace, @Nonnull Path path) throws IOException {
-            return RowIterable.of(namespace, requireNonNull(path, "expected non-null path").toFile());
-        }
-
-        public static RowIterable of(@Nonnull Namespace namespace, @Nonnull String path) throws IOException {
-            return RowIterable.of(namespace, new File(requireNonNull(path, "expected non-null path")));
-        }
-
-        private static class RowIterator implements Iterator<DataItem> {
-
-            final Stack<Utf8String> paths;
-            final Stack<RowReader> readers;
-            final Out value;
-
-            RowReader reader;
-            Result result;
-
-            RowIterator(RowReader reader) {
-                this.readers = new Stack<>();
-                this.paths = new Stack<>();
-                this.reader = reader;
-                this.value = new Out();
-            }
-
-            @Override
-            public boolean hasNext() {
-                return this.reader != null;
-            }
-
-            /**
-             * Returns the next element in the iteration.
-             *
-             * @return the next element in the iteration
-             * @throws NoSuchElementException if the iteration has no more elements
-             */
-            @SuppressWarnings("unchecked")
-            @Override
-            public DataItem next() {
-
-                do {
-                    while (this.reader.read()) {
-
-                        Utf8String path = this.reader.path();
-                        checkState(!path.isNull(), "expected non-null path");
-
-                        LayoutType type = this.reader.type();
-                        checkState(type != null, "expected non-null type");
-
-                        switch (type.layoutCode()) {
-
-                            case BOOLEAN: {
-                                this.result = this.reader.readBoolean(this.value);
-                                break;
-                            }
-                            case INT_16: {
-                                this.result = this.reader.readInt16(this.value);
-                                break;
-                            }
-                            case INT_32: {
-                                this.result = this.reader.readInt32(this.value);
-                                break;
-                            }
-                            case INT_64: {
-                                this.result = this.reader.readInt64(this.value);
-                                break;
-                            }
-                            case UINT_8: {
-                                this.result = this.reader.readUInt8(this.value);
-                                break;
-                            }
-                            case UINT_32: {
-                                this.result = this.reader.readUInt32(this.value);
-                                break;
-                            }
-                            case UINT_64: {
-                                this.result = this.reader.readUInt64(this.value);
-                                break;
-                            }
-                            case BINARY: {
-                                this.result = this.reader.readBinary(this.value);
-                                break;
-                            }
-                            case GUID: {
-                                this.result = this.reader.readGuid(this.value);
-                                break;
-                            }
-                            case NULL:
-                            case BOOLEAN_FALSE:
-                            case INT_8:
-                            case UINT_16:
-                            case VAR_INT:
-                            case VAR_UINT:
-                            case FLOAT_32:
-                            case FLOAT_64:
-                            case FLOAT_128:
-                            case DECIMAL:
-                            case DATE_TIME:
-                            case UNIX_DATE_TIME:
-                            case UTF_8: {
-                                break;
-                            }
-                            case NULLABLE_SCOPE:
-                            case IMMUTABLE_NULLABLE_SCOPE: {
-                                if (!this.reader.hasValue()) {
-                                    break;
-                                }
-                            }
-                            case ARRAY_SCOPE:
-                            case IMMUTABLE_ARRAY_SCOPE:
-
-                            case MAP_SCOPE:
-                            case IMMUTABLE_MAP_SCOPE:
-
-                            case OBJECT_SCOPE:
-                            case IMMUTABLE_OBJECT_SCOPE:
-
-                            case SCHEMA:
-                            case IMMUTABLE_SCHEMA:
-
-                            case SET_SCOPE:
-                            case IMMUTABLE_SET_SCOPE:
-
-                            case TAGGED2_SCOPE:
-                            case IMMUTABLE_TAGGED2_SCOPE:
-
-                            case TAGGED_SCOPE:
-                            case IMMUTABLE_TAGGED_SCOPE:
-
-                            case TUPLE_SCOPE:
-                            case IMMUTABLE_TUPLE_SCOPE:
-
-                            case TYPED_ARRAY_SCOPE:
-                            case IMMUTABLE_TYPED_ARRAY_SCOPE:
-
-                            case TYPED_MAP_SCOPE:
-                            case IMMUTABLE_TYPED_MAP_SCOPE:
-
-                            case TYPED_SET_SCOPE:
-                            case IMMUTABLE_TYPED_SET_SCOPE:
-
-                            case TYPED_TUPLE_SCOPE:
-                            case IMMUTABLE_TYPED_TUPLE_SCOPE: {
-
-                                this.readers.push(this.reader);
-                                this.paths.push(path);
-                                this.reader = this.reader.readScope();
-                                continue;
-                            }
-                            case END_SCOPE: {
-                                final RowReader child = this.reader;
-                                this.reader = this.readers.pop();
-                                continue;
-                            }
-                            case INVALID:
-                            case MONGODB_OBJECT_ID: {
-                                throw new IllegalStateException(lenientFormat("unsupported layout type: %s", type));
-                            }
-                            default: {
-                                throw new IllegalStateException(lenientFormat("unknown layout type: %s", type));
-                            }
-                        }
-
-                        if (this.result != Result.SUCCESS) {
-                            String message = lenientFormat("failed to read %s value for %s", type.layoutCode(), path);
-                            throw new IllegalStateException(message);
-                        }
-
-                        return new DataItem(this.paths, path, type.layoutCode(), this.value.get());
-                    }
-
-                    if (this.readers.empty()) {
-                        this.reader = null;
-                    } else {
-                        this.reader = this.readers.pop();
-                        this.paths.pop();
-                    }
-                }
-                while (this.reader != null);
-
-                throw new NoSuchElementException();
-            }
-        }
-    }
-
 }