random-mirrors/HybridRow
1
0
Fork 0
mirror of https://github.com/microsoft/HybridRow.git synced 2026-01-21 10:23:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

RowIterable now works. RowIterable and RowScanner now include indexes of array elements.

Browse Source
This commit is contained in:
David Noble
2019-09-23 20:30:13 -07:00
parent ee3320f759
commit 6783b569f8
8 changed files with 352 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
java/pom.xml
View File

@@ -18,12 +18,31 @@ Licensed under the MIT License.
<url>https://github.com/Azure/azure-sdk-for-java</url>
<distributionManagement>
<repository>
<name>azure-cosmos-serialization</name>
<layout>default</layout>
<id>dev-azure-com-azure-cosmos-java-azure-cosmos-serialization</id>
<url>https://pkgs.dev.azure.com/azure-cosmos-java/_packaging/azure-cosmos-serialization/maven/v1</url>
</repository>
<site>
<id>azure-java-build-docs</id>
<url>${site.url}/site/${project.artifactId}</url>
</site>
</distributionManagement>
<repositories>
<repository>
<id>dev-azure-com-azure-cosmos-java-azure-cosmos-serialization</id>
<url>https://pkgs.dev.azure.com/azure-cosmos-java/_packaging/azure-cosmos-serialization/maven/v1</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<scm>
<url>https://github.com/Azure/azure-sdk-for-java</url>
</scm>

2
java/src/main/java/com/azure/data/cosmos/core/Out.java
View File

@@ -59,7 +59,7 @@ public final class Out<T> {
return true;
}
if (!(other instanceof Out)) {
if (other.getClass() != Out.class) {
return false;
}

6
java/src/main/java/com/azure/data/cosmos/core/Reference.java
View File

@@ -9,8 +9,8 @@ import java.util.Objects;
* A container object which may or may not contain a non-null value.
*
* This is a value-based class and as such use of identity-sensitive operations--including reference equality
* ({@code ==}), identity hash code, or synchronization--on instances of {@Reference} may have unpredictable results
* and should be avoided.
* ({@code ==}), identity hash code, or synchronization--on instances of {@link Reference} may have unpredictable
* results and should be avoided.
*
* @param <T>
*/
@@ -62,7 +62,7 @@ public final class Reference<T> {
return true;
}
if (!(other instanceof Reference)) {
if (other.getClass() != Reference.class) {
return false;
}

3
java/src/main/java/com/azure/data/cosmos/core/Utf8String.java
View File

@@ -465,6 +465,9 @@ public final class Utf8String implements ByteBufHolder, CharSequence, Comparable
if (this.buffer == null) {
return "null";
}
if (this.buffer.writerIndex() == 0) {
return "\"\"";
}
return Json.toString(this.buffer.getCharSequence(0, this.buffer.writerIndex(), UTF_8));
}

15
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/DataItem.java
View File

@@ -21,9 +21,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
*/
public class DataItem {
private final Supplier<String> fullPath;
private final Supplier<String> name;
@JsonProperty
private final List<String> nodes;
@@ -33,6 +30,9 @@ public class DataItem {
@JsonProperty
private final Object value;
private final Supplier<String> name;
private final Supplier<String> path;
@SuppressWarnings("UnstableApiUsage")
DataItem(
@Nonnull final List<Utf8String> nodes,
@@ -56,7 +56,7 @@ public class DataItem {
this.name = Suppliers.memoize(() -> this.nodes.get(this.nodes.size() - 1));
this.fullPath = Suppliers.memoize(() -> {
this.path = Suppliers.memoize(() -> {
if (this.nodes.size() == 1) {
return this.nodes.get(0);
@@ -69,8 +69,11 @@ public class DataItem {
int i;
for (i = 0; i < this.nodes.size() - 1; i++) {
for (i = 0; i < this.nodes.size() - 1; ++i) {
builder.append(this.nodes.get(i));
if (this.nodes.get(i + 1).charAt(0) != '[') {
builder.append('.');
}
}
return builder.append(this.nodes.get(i)).toString();
@@ -86,7 +89,7 @@ public class DataItem {
}
public String path() {
return this.fullPath.get();
return this.path.get();
}
@Override

293
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowIterable.java Normal file
View File

@@ -0,0 +1,293 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.io;
import com.azure.data.cosmos.core.Out;
import com.azure.data.cosmos.core.Utf8String;
import com.azure.data.cosmos.serialization.hybridrow.HybridRowVersion;
import com.azure.data.cosmos.serialization.hybridrow.Result;
import com.azure.data.cosmos.serialization.hybridrow.RowBuffer;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutResolver;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutResolverNamespace;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutType;
import com.azure.data.cosmos.serialization.hybridrow.schemas.Namespace;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import javax.annotation.Nonnull;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Stack;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.lenientFormat;
import static java.util.Objects.requireNonNull;
public class RowIterable implements AutoCloseable, Iterable<DataItem> {
private AtomicBoolean closed;
private final ByteBuf data;
private final LayoutResolver resolver;
private RowIterable(LayoutResolver resolver, ByteBuf data) {
this.closed = new AtomicBoolean();
this.data = data;
this.resolver = resolver;
}
@Override
public void close() {
if (this.closed.compareAndSet(false, true)) {
this.data.release();
}
}
@Nonnull
@Override
public Iterator<DataItem> iterator() {
checkState(!this.closed.get(), "RowIterable is closed");
final RowBuffer buffer = new RowBuffer(this.data, HybridRowVersion.V1, this.resolver);
final RowReader reader = new RowReader(buffer);
return new RowIterator(reader);
}
public static RowIterable open(@Nonnull Namespace namespace, @Nonnull File file) throws IOException {
checkNotNull(file, "expected non-null file");
final long length = file.length();
checkArgument(0 < length, "file does not exist: %s", file);
checkArgument(length <= Integer.MAX_VALUE, "expected file length <= %s, not %s", Integer.MAX_VALUE, length);
ByteBuf data = Unpooled.buffer((int) length);
try (InputStream stream = Files.newInputStream(file.toPath())) {
data.writeBytes(stream, (int) length);
}
LayoutResolverNamespace resolver = new LayoutResolverNamespace(namespace);
return new RowIterable(resolver, data);
}
public static RowIterable open(@Nonnull Namespace namespace, @Nonnull Path path) throws IOException {
return RowIterable.open(namespace, requireNonNull(path, "expected non-null path").toFile());
}
public static RowIterable open(@Nonnull Namespace namespace, @Nonnull String path) throws IOException {
return RowIterable.open(namespace, new File(requireNonNull(path, "expected non-null path")));
}
private static class RowIterator implements Iterator<DataItem> {
final Stack<Utf8String> paths;
final Stack<RowReader> readers;
final Out value;
DataItem dataItem;
RowReader reader;
RowIterator(RowReader reader) {
this.readers = new Stack<>();
this.paths = new Stack<>();
this.reader = reader;
this.value = new Out();
}
@Override
public boolean hasNext() {
while (this.dataItem == null) {
if (this.reader == null) {
return false;
}
this.advance();
}
return true;
}
/**
* Returns the next element in the iteration.
*
* @return the next element in the iteration
* @throws NoSuchElementException if the iteration has no more elements
*/
@Override
public DataItem next() {
while (this.dataItem == null) {
if (this.reader == null) {
throw new NoSuchElementException();
}
this.advance();
}
DataItem dataItem = this.dataItem;
this.dataItem = null;
return dataItem;
}
@SuppressWarnings("unchecked")
private void advance() {
do {
while (this.reader.read()) {
final Result result;
Utf8String path = this.reader.path();
checkState(!path.isNull(), "expected non-null value for path");
LayoutType type = this.reader.type();
checkState(type != null, "expected non-null type");
switch (type.layoutCode()) {
case BOOLEAN: {
result = this.reader.readBoolean(this.value);
break;
}
case INT_16: {
result = this.reader.readInt16(this.value);
break;
}
case INT_32: {
result = this.reader.readInt32(this.value);
break;
}
case INT_64: {
result = this.reader.readInt64(this.value);
break;
}
case UINT_8: {
result = this.reader.readUInt8(this.value);
break;
}
case UINT_32: {
result = this.reader.readUInt32(this.value);
break;
}
case UINT_64: {
result = this.reader.readUInt64(this.value);
break;
}
case BINARY: {
result = this.reader.readBinary(this.value);
break;
}
case GUID: {
result = this.reader.readGuid(this.value);
break;
}
case NULL:
case BOOLEAN_FALSE:
case INT_8:
case UINT_16:
case VAR_INT:
case VAR_UINT:
case FLOAT_32:
case FLOAT_64:
case FLOAT_128:
case DECIMAL:
case DATE_TIME:
case UNIX_DATE_TIME:
case UTF_8: {
result = Result.SUCCESS;
break;
}
case NULLABLE_SCOPE:
case IMMUTABLE_NULLABLE_SCOPE: {
if (!this.reader.hasValue()) {
result = Result.SUCCESS;
break;
}
}
case ARRAY_SCOPE:
case IMMUTABLE_ARRAY_SCOPE:
case MAP_SCOPE:
case IMMUTABLE_MAP_SCOPE:
case OBJECT_SCOPE:
case IMMUTABLE_OBJECT_SCOPE:
case SCHEMA:
case IMMUTABLE_SCHEMA:
case SET_SCOPE:
case IMMUTABLE_SET_SCOPE:
case TAGGED2_SCOPE:
case IMMUTABLE_TAGGED2_SCOPE:
case TAGGED_SCOPE:
case IMMUTABLE_TAGGED_SCOPE:
case TUPLE_SCOPE:
case IMMUTABLE_TUPLE_SCOPE:
case TYPED_ARRAY_SCOPE:
case IMMUTABLE_TYPED_ARRAY_SCOPE:
case TYPED_MAP_SCOPE:
case IMMUTABLE_TYPED_MAP_SCOPE:
case TYPED_SET_SCOPE:
case IMMUTABLE_TYPED_SET_SCOPE:
case TYPED_TUPLE_SCOPE:
case IMMUTABLE_TYPED_TUPLE_SCOPE: {
this.readers.push(this.reader);
this.paths.push(path.isEmpty()
? Utf8String.transcodeUtf16(lenientFormat("[%s]", this.reader.index()))
: path);
this.reader = this.reader.readScope();
continue;
}
case MONGODB_OBJECT_ID: {
throw new IllegalStateException(lenientFormat("unsupported layout type: %s", type));
}
case END_SCOPE:
case INVALID: {
throw new IllegalStateException(lenientFormat("unexpected layout type: %s", type));
}
default: {
throw new IllegalStateException(lenientFormat("unknown layout type: %s", type));
}
}
if (result != Result.SUCCESS) {
String message = lenientFormat("failed to read %s value for %s", type.layoutCode(), path);
throw new IllegalStateException(message);
}
this.dataItem = new DataItem(this.paths, path, type.layoutCode(), this.value.get());
return;
}
if (this.readers.empty()) {
this.reader = null;
} else {
this.reader = this.readers.pop();
this.paths.pop();
}
}
while (this.reader != null);
}
}
}

5
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowScanner.java
View File

@@ -199,7 +199,10 @@ public class RowScanner implements AutoCloseable {
case TYPED_TUPLE_SCOPE:
case IMMUTABLE_TYPED_TUPLE_SCOPE: {
visitor.nodes().push(path);
visitor.nodes().push(path.isEmpty()
? Utf8String.transcodeUtf16(lenientFormat("[%s]", reader.index()))
: path);
result = reader.readScope(visitor, RowScanner::visit);
visitor.nodes().pop();

254
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/io/RowReaderTest.java
View File

@@ -20,16 +20,12 @@ import org.testng.annotations.Factory;
import org.testng.annotations.Test;
import org.testng.util.Strings;
import javax.annotation.Nonnull;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Stack;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -45,6 +41,8 @@ import static org.testng.AssertJUnit.assertNotNull;
public class RowReaderTest {
// region Fields
private static final String basedir = System.getProperty("project.basedir", System.getProperty("user.dir"));
private final Path dataFile;
@@ -52,6 +50,10 @@ public class RowReaderTest {
private Namespace namespace;
// endregion
// region Construction and Setup
private RowReaderTest(File schemaFile, Path dataFile) {
this.schemaFile = schemaFile;
this.dataFile = dataFile;
@@ -65,13 +67,15 @@ public class RowReaderTest {
);
}
// endregion
@Test(groups = "unit")
public void testIterable() throws IOException {
final RowIterable iterable = RowIterable.of(this.namespace, this.dataFile);
for (DataItem item : iterable) {
assertNotNull(item.toString());
try (final RowIterable iterable = RowIterable.open(this.namespace, this.dataFile)) {
for (DataItem item : iterable) {
assertNotNull(item);
out.println(item);
}
}
}
@@ -105,18 +109,19 @@ public class RowReaderTest {
}
@Test(groups = "unit")
public void testScanner() throws IOException {
final RowScanner scanner = RowScanner.open(this.namespace, this.dataFile);
scanner.visit((DataItem item, Object context) -> {
public void testScanner() throws Exception {
try (final RowScanner scanner = RowScanner.open(this.namespace, this.dataFile)) {
scanner.visit((DataItem item, Object context) -> {
assertNull(context);
assertNotNull(item);
out.println(item);
return Result.SUCCESS;
}, null);
}
}
// region Privates
@SuppressWarnings("unchecked")
private static Result visitFields(RowReader reader, int level) {
@@ -281,224 +286,5 @@ public class RowReaderTest {
}
}
public static class RowIterable implements Iterable<DataItem> {
final ByteBuf data;
final LayoutResolver resolver;
private RowIterable(LayoutResolver resolver, ByteBuf data) {
this.resolver = resolver;
this.data = data;
}
@Nonnull
@Override
public Iterator<DataItem> iterator() {
final RowBuffer buffer = new RowBuffer(this.data, HybridRowVersion.V1, this.resolver);
final RowReader reader = new RowReader(buffer);
return new RowIterator(reader);
}
public static RowIterable of(@Nonnull Namespace namespace, @Nonnull File file) throws IOException {
checkNotNull(file, "expected non-null file");
final long length = file.length();
checkArgument(0 < length, "file does not exist: %s", file);
checkArgument(length <= Integer.MAX_VALUE, "expected file length <= %s, not %s", Integer.MAX_VALUE, length);
ByteBuf data = Unpooled.buffer((int) length);
try (InputStream stream = Files.newInputStream(file.toPath())) {
data.writeBytes(stream, (int) length);
}
LayoutResolverNamespace resolver = new LayoutResolverNamespace(namespace);
return new RowIterable(resolver, data);
}
public static RowIterable of(@Nonnull Namespace namespace, @Nonnull Path path) throws IOException {
return RowIterable.of(namespace, requireNonNull(path, "expected non-null path").toFile());
}
public static RowIterable of(@Nonnull Namespace namespace, @Nonnull String path) throws IOException {
return RowIterable.of(namespace, new File(requireNonNull(path, "expected non-null path")));
}
private static class RowIterator implements Iterator<DataItem> {
final Stack<Utf8String> paths;
final Stack<RowReader> readers;
final Out value;
RowReader reader;
Result result;
RowIterator(RowReader reader) {
this.readers = new Stack<>();
this.paths = new Stack<>();
this.reader = reader;
this.value = new Out();
}
@Override
public boolean hasNext() {
return this.reader != null;
}
/**
* Returns the next element in the iteration.
*
* @return the next element in the iteration
* @throws NoSuchElementException if the iteration has no more elements
*/
@SuppressWarnings("unchecked")
@Override
public DataItem next() {
do {
while (this.reader.read()) {
Utf8String path = this.reader.path();
checkState(!path.isNull(), "expected non-null path");
LayoutType type = this.reader.type();
checkState(type != null, "expected non-null type");
switch (type.layoutCode()) {
case BOOLEAN: {
this.result = this.reader.readBoolean(this.value);
break;
}
case INT_16: {
this.result = this.reader.readInt16(this.value);
break;
}
case INT_32: {
this.result = this.reader.readInt32(this.value);
break;
}
case INT_64: {
this.result = this.reader.readInt64(this.value);
break;
}
case UINT_8: {
this.result = this.reader.readUInt8(this.value);
break;
}
case UINT_32: {
this.result = this.reader.readUInt32(this.value);
break;
}
case UINT_64: {
this.result = this.reader.readUInt64(this.value);
break;
}
case BINARY: {
this.result = this.reader.readBinary(this.value);
break;
}
case GUID: {
this.result = this.reader.readGuid(this.value);
break;
}
case NULL:
case BOOLEAN_FALSE:
case INT_8:
case UINT_16:
case VAR_INT:
case VAR_UINT:
case FLOAT_32:
case FLOAT_64:
case FLOAT_128:
case DECIMAL:
case DATE_TIME:
case UNIX_DATE_TIME:
case UTF_8: {
break;
}
case NULLABLE_SCOPE:
case IMMUTABLE_NULLABLE_SCOPE: {
if (!this.reader.hasValue()) {
break;
}
}
case ARRAY_SCOPE:
case IMMUTABLE_ARRAY_SCOPE:
case MAP_SCOPE:
case IMMUTABLE_MAP_SCOPE:
case OBJECT_SCOPE:
case IMMUTABLE_OBJECT_SCOPE:
case SCHEMA:
case IMMUTABLE_SCHEMA:
case SET_SCOPE:
case IMMUTABLE_SET_SCOPE:
case TAGGED2_SCOPE:
case IMMUTABLE_TAGGED2_SCOPE:
case TAGGED_SCOPE:
case IMMUTABLE_TAGGED_SCOPE:
case TUPLE_SCOPE:
case IMMUTABLE_TUPLE_SCOPE:
case TYPED_ARRAY_SCOPE:
case IMMUTABLE_TYPED_ARRAY_SCOPE:
case TYPED_MAP_SCOPE:
case IMMUTABLE_TYPED_MAP_SCOPE:
case TYPED_SET_SCOPE:
case IMMUTABLE_TYPED_SET_SCOPE:
case TYPED_TUPLE_SCOPE:
case IMMUTABLE_TYPED_TUPLE_SCOPE: {
this.readers.push(this.reader);
this.paths.push(path);
this.reader = this.reader.readScope();
continue;
}
case END_SCOPE: {
final RowReader child = this.reader;
this.reader = this.readers.pop();
continue;
}
case INVALID:
case MONGODB_OBJECT_ID: {
throw new IllegalStateException(lenientFormat("unsupported layout type: %s", type));
}
default: {
throw new IllegalStateException(lenientFormat("unknown layout type: %s", type));
}
}
if (this.result != Result.SUCCESS) {
String message = lenientFormat("failed to read %s value for %s", type.layoutCode(), path);
throw new IllegalStateException(message);
}
return new DataItem(this.paths, path, type.layoutCode(), this.value.get());
}
if (this.readers.empty()) {
this.reader = null;
} else {
this.reader = this.readers.pop();
this.paths.pop();
}
}
while (this.reader != null);
throw new NoSuchElementException();
}
}
}
// endregion
}