random-mirrors/HybridRow
1
0
Fork 0
mirror of https://github.com/microsoft/HybridRow.git synced 2026-01-20 18:03:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Progressed on RowReader

Browse Source
This commit is contained in:
David Noble
2019-09-19 22:55:25 -07:00
parent 840532aa03
commit 66a6d20a30
17 changed files with 536 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
java/src/main/java/com/azure/data/cosmos/core/Json.java
View File

@@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
@@ -36,11 +37,20 @@ public final class Json {
private Json() {
}
public static <T> Optional<T> parse(File file, Class<T> type) {
try {
return Optional.of(reader.forType(type).readValue(file));
} catch (IOException error) {
logger.error("failed to parse {} due to ", type.getName(), error);
return Optional.empty();
}
}
public static <T> Optional<T> parse(InputStream stream, Class<T> type) {
try {
return Optional.of(reader.forType(type).readValue(stream));
} catch (IOException error) {
logger.error("failed to parse %s due to ", type.getName(), error);
logger.error("failed to parse {} due to ", type.getName(), error);
return Optional.empty();
}
}

9
java/src/main/java/com/azure/data/cosmos/core/Utf8String.java
View File

@@ -457,10 +457,17 @@ public final class Utf8String implements ByteBufHolder, CharSequence, Comparable
@Override
@Nonnull
public String toString() {
return this.buffer.getCharSequence(0, this.buffer.writerIndex(), UTF_8).toString();
if (this.buffer == null) {
return "null";
}
return Json.toString(this.buffer.getCharSequence(0, this.buffer.writerIndex(), UTF_8));
}
@Nullable
public String toUtf16() {
if (this.buffer == null) {
return null;
}
return this.buffer.getCharSequence(0, this.buffer.writerIndex(), UTF_8).toString();
}

20
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/HybridRowHeader.java
View File

@@ -3,6 +3,10 @@
package com.azure.data.cosmos.serialization.hybridrow;
import javax.annotation.Nonnull;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Describes the header that precedes all valid Hybrid Rows.
*/
@@ -10,10 +14,10 @@ public final class HybridRowHeader {
/**
* Size (in bytes) of a serialized header.
*/
public static final int BYTES = SchemaId.BYTES;
public static final int BYTES = HybridRowVersion.BYTES + SchemaId.BYTES;
private SchemaId schemaId;
private HybridRowVersion version = HybridRowVersion.values()[0];
private final SchemaId schemaId;
private final HybridRowVersion version;
/**
* Initializes a new instance of a {@link HybridRowHeader}.
@@ -21,9 +25,13 @@ public final class HybridRowHeader {
* @param version The version of the HybridRow library used to write this row.
* @param schemaId The unique identifier of the schema whose layout was used to write this row.
*/
public HybridRowHeader(HybridRowVersion version, SchemaId schemaId) {
public HybridRowHeader(@Nonnull final HybridRowVersion version, @Nonnull SchemaId schemaId) {
checkNotNull(version, "expected non-null version");
checkNotNull(schemaId, "expected non-null schemaId");
this.version = version;
this.schemaId = SchemaId.from(schemaId.value());
this.schemaId = schemaId;
}
/**
@@ -31,6 +39,7 @@ public final class HybridRowHeader {
*
* @return unique identifier of the schema whose layout was used to write this {@link HybridRowHeader}.
*/
@Nonnull
public SchemaId schemaId() {
return this.schemaId;
}
@@ -40,6 +49,7 @@ public final class HybridRowHeader {
*
* @return version of the HybridRow serialization library used to write this {@link HybridRowHeader}.
*/
@Nonnull
public HybridRowVersion version() {
return this.version;
}

55
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/RowBuffer.java
View File

@@ -828,34 +828,37 @@ public final class RowBuffer {
}
/**
* Read the value of a {@code SparseLen} field at the given {@code offset} position.
* Read the value of a {@code SparsePathLen} field at the given {@code offset} position.
*
* @param layout
* @param offset position of a {@code SparseLen} field within this {@link RowBuffer}.
* @param pathLenInBytes
* @param pathOffset
* @return the {@code SparseLen} value read.
* @param layout layout of the {@code SparsePathLen} field.
* @param offset position of the {@code SparsePathLen} field..
* @param pathOffset [output] position of the {@code SparsePathLen} field value.
* @param pathLengthInBytes [output] length of the {@code SparsePathLen} field value.
* @return the {@code SparsePathLen} value read.
*/
public int readSparsePathLen(
@Nonnull final Layout layout, final int offset, @Nonnull final Out<Integer> pathLenInBytes,
@Nonnull final Out<Integer> pathOffset) {
@Nonnull final Layout layout,
final int offset,
@Nonnull final Out<Integer> pathOffset,
@Nonnull final Out<Integer> pathLengthInBytes) {
checkNotNull(layout, "expected non-null layout");
checkNotNull(pathOffset, "expected non-null pathOffset");
checkNotNull(pathLenInBytes, "expected non-null pathLenInBytes");
checkNotNull(pathLengthInBytes, "expected non-null pathLengthInBytes");
final Item<Long> item = this.read(this::read7BitEncodedUInt, offset);
final int token = item.value().intValue();
if (token < layout.tokenizer().count()) {
pathLenInBytes.set(item.length());
pathLengthInBytes.set(item.length());
pathOffset.set(offset);
return token;
}
final int numBytes = token - layout.tokenizer().count();
pathLenInBytes.set(numBytes + item.length());
pathLengthInBytes.set(numBytes + item.length());
pathOffset.set(offset + item.length());
return token;
}
@@ -878,7 +881,10 @@ public final class RowBuffer {
* @return the {@code SparseTypeCode} value read.
*/
public LayoutType readSparseTypeCode(int offset) {
return LayoutType.fromCode(LayoutCode.from(this.readInt8(offset)));
byte value = this.readInt8(offset);
LayoutCode code = LayoutCode.from(value);
checkState(code != null, "expected layout code at offset %s, not %s", offset, code);
return LayoutType.fromLayoutCode(code);
}
/**
@@ -1191,8 +1197,8 @@ public final class RowBuffer {
this.readSparseMetadata(edit);
// Check if reached end of sparse scope.
if (!(edit.cellType() instanceof LayoutEndScope)) {
// End of sparse scope
edit.exists(true);
return true;
}
@@ -2978,7 +2984,7 @@ public final class RowBuffer {
for (int i = 1; i < uniqueIndex.size(); i++) {
UniqueIndexItem x = uniqueIndex.get(i);
leftEdit.cellType(LayoutType.fromCode(x.code()));
leftEdit.cellType(LayoutType.fromLayoutCode(x.code()));
leftEdit.metaOffset(x.metaOffset());
leftEdit.valueOffset(x.valueOffset());
@@ -2988,7 +2994,7 @@ public final class RowBuffer {
int j;
for (j = i - 1; j >= 0; j--) {
UniqueIndexItem y = uniqueIndex.get(j);
rightEdit.cellType(LayoutType.fromCode(y.code()));
rightEdit.cellType(LayoutType.fromLayoutCode(y.code()));
rightEdit.metaOffset(y.metaOffset());
rightEdit.valueOffset(y.valueOffset());
@@ -3144,12 +3150,19 @@ public final class RowBuffer {
checkNotNull(edit, "expected non-null edit");
if (edit.scopeType().hasImplicitTypeCode(edit)) {
edit.scopeType().setImplicitTypeCode(edit);
edit.valueOffset(edit.metaOffset());
} else {
edit.cellType(this.readSparseTypeCode(edit.metaOffset()));
edit.valueOffset(edit.metaOffset() + LayoutCode.BYTES);
int metaOffset = edit.metaOffset();
LayoutType layoutType = this.readSparseTypeCode(metaOffset);
edit.cellType(layoutType);
edit.valueOffset(metaOffset + LayoutCode.BYTES);
edit.cellTypeArgs(TypeArgumentList.EMPTY);
if (edit.cellType() instanceof LayoutEndScope) {
// Reached end of current scope without finding another field.
edit.pathToken(0);
@@ -3158,9 +3171,9 @@ public final class RowBuffer {
return;
}
Out<Integer> sizeLenInBytes = new Out<>();
edit.cellTypeArgs(edit.cellType().readTypeArgumentList(this, edit.valueOffset(), sizeLenInBytes));
edit.valueOffset(edit.valueOffset() + sizeLenInBytes.get());
Out<Integer> lengthInBytes = new Out<>();
edit.cellTypeArgs(edit.cellType().readTypeArgumentList(this, edit.valueOffset(), lengthInBytes));
edit.valueOffset(edit.valueOffset() + lengthInBytes.get());
}
edit.scopeType().readSparsePath(this, edit);
@@ -3203,7 +3216,7 @@ public final class RowBuffer {
int offset = edit.metaOffset() + LayoutCode.BYTES;
Out<Integer> pathLenInBytes = new Out<>();
Out<Integer> pathOffset = new Out<>();
int token = this.readSparsePathLen(edit.layout(), offset, pathLenInBytes, pathOffset);
int token = this.readSparsePathLen(edit.layout(), offset, pathOffset, pathLenInBytes);
checkState(edit.pathOffset() == pathOffset.get());
checkState(edit.pathToken() == token);
}

7
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/RowCursor.java
View File

@@ -112,16 +112,15 @@ public final class RowCursor implements Cloneable {
final SchemaId schemaId = row.readSchemaId(1);
final Layout layout = row.resolver().resolve(schemaId);
final int sparseSegmentOffset = row.computeVariableValueOffset(layout, HybridRowHeader.BYTES,
layout.numVariable());
final int offset = row.computeVariableValueOffset(layout, HybridRowHeader.BYTES, layout.numVariable());
return new RowCursor()
.layout(layout)
.scopeType(LayoutTypes.UDT)
.scopeTypeArgs(new TypeArgumentList(schemaId))
.start(HybridRowHeader.BYTES)
.metaOffset(sparseSegmentOffset)
.valueOffset(sparseSegmentOffset);
.metaOffset(offset)
.valueOffset(offset);
}
public static RowCursor createForAppend(RowBuffer row) {

105
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowReader.java
View File

@@ -5,7 +5,6 @@ package com.azure.data.cosmos.serialization.hybridrow.io;
import com.azure.data.cosmos.core.Out;
import com.azure.data.cosmos.core.Utf8String;
import com.azure.data.cosmos.core.UtfAnyString;
import com.azure.data.cosmos.serialization.hybridrow.Float128;
import com.azure.data.cosmos.serialization.hybridrow.NullValue;
import com.azure.data.cosmos.serialization.hybridrow.Result;
@@ -51,6 +50,7 @@ import java.time.OffsetDateTime;
import java.util.List;
import java.util.UUID;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.lenientFormat;
@@ -79,19 +79,22 @@ public final class RowReader {
*
* @param row The row to be read
*/
public RowReader(RowBuffer row) {
public RowReader(@Nonnull RowBuffer row) {
this(row, RowCursor.create(row));
}
/**
* Initializes a new instance of the {@link RowReader} class.
*
* @param row The row to be read
* @param buffer The buffer to be read
* @param checkpoint Initial state of the reader
*/
public RowReader(@Nonnull final RowBuffer row, @Nonnull final Checkpoint checkpoint) {
public RowReader(@Nonnull final RowBuffer buffer, @Nonnull final Checkpoint checkpoint) {
this.row = row;
checkNotNull(buffer, "expected non-null buffer");
checkNotNull(checkpoint, "expected non-null checkpoint");
this.row = buffer;
this.columns = checkpoint.cursor().layout().columns();
this.schematizedCount = checkpoint.cursor().layout().numFixed() + checkpoint.cursor().layout().numVariable();
@@ -103,18 +106,21 @@ public final class RowReader {
/**
* Initializes a new instance of the {@link RowReader} class.
*
* @param row The row to be read
* @param scope Cursor defining the scope of the fields to be read
* <p>
* A {@link RowReader} instance traverses all of the top-level fields of a given scope. If the root
* scope is provided then all top-level fields in the row are enumerated. Nested child
* {@link RowReader} instances can be access through the {@link RowReader#readScope} method to process
* nested content.
* @param buffer The buffer to be read.
* @param scope Cursor defining the scope of the fields to be read.
* <p>
* A {@link RowReader} instance traverses all of the top-level fields of a given scope. If the root
* scope is provided then all top-level fields in the buffer are enumerated. Nested child
* {@link RowReader} instances can be access through the {@link RowReader#readScope} method to process
* nested content.
*/
private RowReader(@Nonnull final RowBuffer row, @Nonnull final RowCursor scope) {
private RowReader(@Nonnull final RowBuffer buffer, @Nonnull final RowCursor scope) {
checkNotNull(buffer, "expected non-null buffer");
checkNotNull(scope, "expected non-null scope");
this.cursor = scope;
this.row = row;
this.row = buffer;
this.columns = this.cursor.layout().columns();
this.schematizedCount = this.cursor.layout().numFixed() + this.cursor.layout().numVariable();
@@ -125,7 +131,7 @@ public final class RowReader {
/**
* Read the current field as a fixed length {@code MongoDbObjectId} value.
*
* @param value On success, receives the value, undefined otherwise
* @param value On success, receives the value, undefined otherwise.
* @return {@link Result#SUCCESS} if the read is successful, an error {@link Result} otherwise.
*/
public Result ReadMongoDbObjectId(Out<?/* MongoDbObjectID */> value) {
@@ -184,7 +190,7 @@ public final class RowReader {
* When enumerating a non-indexed scope, this value is always zero.
*
* @return zero-based index of the field relative to the scope, if positioned on a field; otherwise undefined.
* @see #path
* @see #path()
*/
public int index() {
return this.state == States.SPARSE ? this.cursor.index() : 0;
@@ -199,36 +205,6 @@ public final class RowReader {
return this.row.length();
}
/**
* The path, relative to the scope, of the field--if positioned on a field--undefined otherwise.
* <p>
* When enumerating an indexed scope, this value is always {@code null}.
*
* @return path of the field relative to the scope, if positioned on a field; otherwise undefined.
* @see #index
*/
public UtfAnyString path() {
Utf8String value;
switch (this.state) {
case SCHEMATIZED:
value = this.columns.get(this.columnIndex).path();
break;
case SPARSE:
value = this.cursor.pathOffset() == 0 ? null : this.row.readSparsePath(this.cursor);
break;
default:
value = null;
break;
}
return new UtfAnyString(value);
}
/**
* The path, relative to the scope, of the field--if positioned on a field--undefined otherwise.
* <p>
@@ -237,14 +213,15 @@ public final class RowReader {
* @return path of the field relative to the scope, if positioned on a field; otherwise undefined.
* @see #index
*/
public Utf8String pathSpan() {
@Nonnull
public Utf8String path() {
switch (this.state) {
case SCHEMATIZED:
return this.columns.get(this.columnIndex).path();
case SPARSE:
return this.row.readSparsePath(this.cursor);
default:
return null;
return Utf8String.NULL;
}
}
@@ -255,7 +232,7 @@ public final class RowReader {
*/
public boolean read() {
for (; ; ) {
while (this.state != States.DONE) {
switch (this.state) {
@@ -268,20 +245,18 @@ public final class RowReader {
this.columnIndex++;
if (this.columnIndex >= this.schematizedCount) {
this.state = States.SPARSE;
} else {
checkState(this.cursor.scopeType() instanceof LayoutUDT);
LayoutColumn column = this.columns.get(this.columnIndex);
if (!this.row.readBit(this.cursor.start(), column.nullBit())) {
break; // to skip schematized values if they aren't present
}
return true;
break;
}
checkState(this.cursor.scopeType() instanceof LayoutUDT);
LayoutColumn column = this.columns.get(this.columnIndex);
if (!this.row.readBit(this.cursor.start(), column.nullBit())) {
break; // to skip schematized values if they aren't present
}
return true;
}
case SPARSE: {
@@ -291,11 +266,10 @@ public final class RowReader {
}
return true;
}
case DONE: {
return false;
}
}
}
return false;
}
/**
@@ -987,8 +961,9 @@ public final class RowReader {
/**
* The type of the field--if positioned on a field--undefined otherwise.
*
* @return layout type.
* @return layout type or {@code null}.
*/
@Nullable
public LayoutType type() {
switch (this.state) {

40
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/IRowSerializable.java → java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowSerializable.java
View File

@@ -1,21 +1,21 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.io;
import com.azure.data.cosmos.serialization.hybridrow.Result;
import com.azure.data.cosmos.serialization.hybridrow.layouts.TypeArgument;
/**
* A type may implement this interface to support serialization into a HybridRow.
*/
public interface IRowSerializable {
/**
* Writes the current instance into the row
*
* @param writer A writer for the current row scope
* @param typeArg The schematized layout type, if a schema is available
* @return Success if the write is successful, the error code otherwise
*/
Result write(RowWriter writer, TypeArgument typeArg);
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.io;
import com.azure.data.cosmos.serialization.hybridrow.Result;
import com.azure.data.cosmos.serialization.hybridrow.layouts.TypeArgument;
/**
* A type may implement this interface to support serialization into a HybridRow.
*/
public interface RowSerializable {
/**
* Writes the current instance into the row
*
* @param writer A writer for the current row scope
* @param typeArg The schematized layout type, if a schema is available
* @return Success if the write is successful, the error code otherwise
*/
Result write(RowWriter writer, TypeArgument typeArg);
}

2
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/LayoutCode.java
View File

@@ -7,6 +7,7 @@ import com.google.common.base.Suppliers;
import it.unimi.dsi.fastutil.bytes.Byte2ReferenceMap;
import it.unimi.dsi.fastutil.bytes.Byte2ReferenceOpenHashMap;
import javax.annotation.Nullable;
import java.util.function.Supplier;
/**
@@ -111,6 +112,7 @@ public enum LayoutCode {
return this.value;
}
@Nullable
public static LayoutCode from(final byte value) {
return mappings.get().get(value);
}

37
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/LayoutCompiler.java
View File

@@ -20,6 +20,7 @@ import com.azure.data.cosmos.serialization.hybridrow.schemas.TuplePropertyType;
import com.azure.data.cosmos.serialization.hybridrow.schemas.TypeKind;
import com.azure.data.cosmos.serialization.hybridrow.schemas.UdtPropertyType;
import com.google.common.base.Strings;
import org.checkerframework.checker.index.qual.NonNegative;
import javax.annotation.Nonnull;
import java.util.List;
@@ -36,32 +37,36 @@ public final class LayoutCompiler {
/**
* Compiles a logical schema into a physical layout that can be used to read and write rows
*
* @param ns The namespace within which {@code schema} is defined
* @param namespace The namespace within which {@code schema} is defined
* @param schema The logical schema to produce a layout for
* @return The layout for the schema
*/
@Nonnull
public static Layout compile(@Nonnull final Namespace ns, @Nonnull final Schema schema) {
public static Layout compile(@Nonnull final Namespace namespace, @Nonnull final Schema schema) {
checkNotNull(ns, "expected non-null ns");
checkNotNull(namespace, "expected non-null namespace");
checkNotNull(schema, "expected non-null schema");
checkArgument(schema.type() == TypeKind.SCHEMA);
checkArgument(!Strings.isNullOrEmpty(schema.name()));
checkArgument(ns.schemas().contains(schema));
checkArgument(namespace.schemas().contains(schema));
LayoutBuilder builder = new LayoutBuilder(schema.name(), schema.schemaId());
LayoutCompiler.addProperties(builder, ns, LayoutCode.SCHEMA, schema.properties());
LayoutCompiler.addProperties(builder, namespace, LayoutCode.SCHEMA, schema.properties());
return builder.build();
}
private static void addProperties(LayoutBuilder builder, Namespace ns, LayoutCode scope, List<Property> properties) {
private static void addProperties(
@Nonnull final LayoutBuilder builder,
@Nonnull final Namespace namespace,
@Nonnull LayoutCode layoutCode,
@Nonnull List<Property> properties) {
final Out<TypeArgumentList> typeArgs = new Out<>();
for (Property p : properties) {
LayoutType type = LayoutCompiler.logicalToPhysicalType(ns, p.type(), typeArgs);
LayoutType type = LayoutCompiler.logicalToPhysicalType(namespace, p.type(), typeArgs);
switch (LayoutCodeTraits.clearImmutableBit(type.layoutCode())) {
@@ -71,7 +76,7 @@ public final class LayoutCompiler {
}
ObjectPropertyType op = (ObjectPropertyType)p.type();
builder.addObjectScope(p.path(), type);
LayoutCompiler.addProperties(builder, ns, type.layoutCode(), op.properties());
LayoutCompiler.addProperties(builder, namespace, type.layoutCode(), op.properties());
builder.EndObjectScope();
break;
}
@@ -107,9 +112,9 @@ public final class LayoutCompiler {
switch (pp.storage()) {
case FIXED:
if (LayoutCodeTraits.clearImmutableBit(scope) != LayoutCode.SCHEMA) {
if (LayoutCodeTraits.clearImmutableBit(layoutCode) != LayoutCode.SCHEMA) {
throw new LayoutCompilationException(
"Cannot have fixed storage within a sparse scope.");
"Cannot have fixed storage within a sparse layoutCode.");
}
if (type.isNull() && !pp.nullable()) {
throw new LayoutCompilationException(
@@ -119,9 +124,9 @@ public final class LayoutCompiler {
break;
case VARIABLE:
if (LayoutCodeTraits.clearImmutableBit(scope) != LayoutCode.SCHEMA) {
if (LayoutCodeTraits.clearImmutableBit(layoutCode) != LayoutCode.SCHEMA) {
throw new LayoutCompilationException(
"Cannot have variable storage within a sparse scope.");
"Cannot have variable storage within a sparse layoutCode.");
}
if (!pp.nullable()) {
throw new LayoutCompilationException(
@@ -395,7 +400,7 @@ public final class LayoutCompiler {
final Optional<Schema> udtSchema;
if (up.schemaId() == SchemaId.INVALID) {
if (up.schemaId() == SchemaId.NONE) {
udtSchema = namespace.schemas().stream()
.filter(schema -> up.name().equals(schema.name()))
.findFirst();
@@ -403,16 +408,16 @@ public final class LayoutCompiler {
udtSchema = namespace.schemas().stream()
.filter(schema -> up.schemaId().equals(schema.schemaId()))
.findFirst();
if (up.name().equals(udtSchema.map(Schema::name).orElse(null))) {
if (udtSchema.isPresent() && !up.name().equals(udtSchema.get().name())) {
throw new LayoutCompilationException(lenientFormat(
"Ambiguous schema reference: '%s:%s'", up.name(), up.schemaId()
"ambiguous schema reference: '%s:%s'", up.name(), up.schemaId()
));
}
}
if (!udtSchema.isPresent()) {
throw new LayoutCompilationException(lenientFormat(
"Cannot resolve schema reference '%s:%s'", up.name(), up.schemaId()
"cannot resolve schema reference '%s:%s'", up.name(), up.schemaId()
));
}

25
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/LayoutType.java
View File

@@ -3,10 +3,12 @@
package com.azure.data.cosmos.serialization.hybridrow.layouts;
import com.azure.data.cosmos.core.Json;
import com.azure.data.cosmos.core.Out;
import com.azure.data.cosmos.serialization.hybridrow.Result;
import com.azure.data.cosmos.serialization.hybridrow.RowBuffer;
import com.azure.data.cosmos.serialization.hybridrow.RowCursor;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.annotation.Nonnull;
@@ -24,9 +26,16 @@ public abstract class LayoutType /*implements ILayoutType*/ {
private static final LayoutType[] codeIndex = new LayoutType[LayoutCode.END_SCOPE.value() + 1];
@JsonProperty
private final boolean immutable;
@JsonProperty
private final LayoutCode layoutCode;
@JsonProperty
private final int size;
@JsonProperty
private final TypeArgument typeArg;
/**
@@ -36,8 +45,8 @@ public abstract class LayoutType /*implements ILayoutType*/ {
checkNotNull(code, "expected non-null code");
this.layoutCode = code;
this.immutable = immutable;
this.layoutCode = code;
this.size = size;
this.typeArg = new TypeArgument(this);
@@ -97,9 +106,9 @@ public abstract class LayoutType /*implements ILayoutType*/ {
}
@Nonnull
public static LayoutType fromCode(LayoutCode code) {
public static LayoutType fromLayoutCode(LayoutCode code) {
LayoutType type = LayoutType.codeIndex[code.value()];
assert type != null : lenientFormat("Not Implemented: %s", code);
checkArgument(type != null, "unimplemented code: %s", code);
return type;
}
@@ -335,6 +344,16 @@ public abstract class LayoutType /*implements ILayoutType*/ {
return this.size;
}
/**
* Returns a string representation of the {@linkplain LayoutType layout type}.
*
* @return a string representation of the {@linkplain LayoutType layout type}.
*/
@Override
public String toString() {
return Json.toString(this);
}
public TypeArgument typeArg() {
return this.typeArg;
}

6
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/LayoutTypeScope.java
View File

@@ -133,11 +133,11 @@ public abstract class LayoutTypeScope extends LayoutType {
}
public void readSparsePath(@Nonnull final RowBuffer buffer, @Nonnull final RowCursor edit) {
Out<Integer> pathLenInBytes = new Out<>();
Out<Integer> pathLengthInBytes = new Out<>();
Out<Integer> pathOffset = new Out<>();
edit.pathToken(buffer.readSparsePathLen(edit.layout(), edit.valueOffset(), pathLenInBytes, pathOffset));
edit.pathToken(buffer.readSparsePathLen(edit.layout(), edit.valueOffset(), pathOffset, pathLengthInBytes));
edit.pathOffset(pathOffset.get());
edit.valueOffset(edit.valueOffset() + pathLenInBytes.get());
edit.valueOffset(edit.valueOffset() + pathLengthInBytes.get());
}
public void setImplicitTypeCode(@Nonnull final RowCursor edit) {

52
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/TypeArgument.java
View File

@@ -3,26 +3,29 @@
package com.azure.data.cosmos.serialization.hybridrow.layouts;
import com.azure.data.cosmos.core.Json;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.annotation.Nonnull;
import static com.google.common.base.Preconditions.checkNotNull;
public final class TypeArgument {
public static final TypeArgument NONE = new TypeArgument();
private final LayoutType type;
private final TypeArgumentList typeArgs;
private TypeArgument() {
this.type = null;
this.typeArgs = null;
}
private final LayoutType type;
@JsonProperty
private final TypeArgumentList typeArgs;
/**
* Initializes a new instance of the {@link TypeArgument} struct.
*
* @param type The type of the constraint.
*/
public TypeArgument(LayoutType type) {
checkNotNull(type);
public TypeArgument(@Nonnull LayoutType type) {
checkNotNull(type, "expected non-null type");
this.type = type;
this.typeArgs = TypeArgumentList.EMPTY;
}
@@ -33,37 +36,38 @@ public final class TypeArgument {
* @param type The type of the constraint.
* @param typeArgs For generic types the type parameters.
*/
public TypeArgument(LayoutType type, TypeArgumentList typeArgs) {
checkNotNull(type);
public TypeArgument(@Nonnull LayoutType type, @Nonnull TypeArgumentList typeArgs) {
checkNotNull(type, "expected non-null type");
checkNotNull(type, "expected non-null typeArgs");
this.type = type;
this.typeArgs = typeArgs;
}
@Override
public boolean equals(Object other) {
if (null == other) {
return false;
}
return other instanceof TypeArgument && this.equals((TypeArgument) other);
private TypeArgument() {
this.type = null;
this.typeArgs = null;
}
@Override
public int hashCode() {
return (this.type.hashCode() * 397) ^ this.typeArgs.hashCode();
public boolean equals(Object other) {
if (null == other) {
return false;
}
return other.getClass() == TypeArgument.class && this.equals((TypeArgument) other);
}
public boolean equals(TypeArgument other) {
return this.type.equals(other.type) && this.typeArgs.equals(other.typeArgs);
}
@Override
public int hashCode() {
return (this.type.hashCode() * 397) ^ this.typeArgs.hashCode();
}
@Override
public String toString() {
if (this.type == null) {
return "";
}
return this.type.name() + this.typeArgs.toString();
return Json.toString(this);
}
/**

1
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/TypeArgumentList.java
View File

@@ -5,6 +5,7 @@ package com.azure.data.cosmos.serialization.hybridrow.layouts;
import com.azure.data.cosmos.core.Json;
import com.azure.data.cosmos.serialization.hybridrow.SchemaId;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;

74
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/schemas/Namespace.java
View File

@@ -5,9 +5,12 @@ package com.azure.data.cosmos.serialization.hybridrow.schemas;
import com.azure.data.cosmos.core.Json;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
@@ -46,6 +49,46 @@ public class Namespace {
return this;
}
/**
* Parse a JSON document and return a full namespace.
*
* @param file The JSON file to parse.
* @return A namespace containing a set of logical schemas.
*/
public static Optional<Namespace> parse(File file) {
Optional<Namespace> namespace = Json.parse(file, Namespace.class);
namespace.ifPresent(SchemaValidator::validate);
return namespace;
}
/**
* Parse a JSON document and return a full namespace.
*
* @param stream The JSON input stream to parse.
* @return A namespace containing a set of logical schemas.
*/
public static Optional<Namespace> parse(InputStream stream) {
Optional<Namespace> namespace = Json.parse(stream, Namespace.class);
try {
namespace.ifPresent(SchemaValidator::validate);
} catch (SchemaException error) {
logger.error("failed to parse {} due to ", Namespace.class, error);
}
return namespace;
}
/**
* Parse a JSON document and return a full namespace.
*
* @param value The JSON text to parse.
* @return A namespace containing a set of logical schemas.
*/
public static Optional<Namespace> parse(String value) {
Optional<Namespace> namespace = Json.parse(value, Namespace.class);
namespace.ifPresent(SchemaValidator::validate);
return namespace;
}
/**
* The set of schemas that make up the {@link Namespace}.
* <p>
@@ -81,36 +124,9 @@ public class Namespace {
* used to encode this {@linkplain Namespace namespace}.
* @return a reference to this {@linkplain Namespace namespace}.
*/
public final Namespace version(SchemaLanguageVersion value) {
public final Namespace version(@Nonnull SchemaLanguageVersion value) {
Preconditions.checkNotNull(value, "expected non-null value");
this.version = value;
return this;
}
/**
* Parse a JSON document and return a full namespace.
*
* @param value The JSON text to parse.
* @return A namespace containing a set of logical schemas.
*/
public static Optional<Namespace> parse(String value) {
Optional<Namespace> namespace = Json.parse(value, Namespace.class);
namespace.ifPresent(SchemaValidator::validate);
return namespace;
}
/**
* Parse a JSON document and return a full namespace.
*
* @param stream The JSON input stream to parse.
* @return A namespace containing a set of logical schemas.
*/
public static Optional<Namespace> parse(InputStream stream) {
Optional<Namespace> namespace = Json.parse(stream, Namespace.class);
try {
namespace.ifPresent(SchemaValidator::validate);
} catch (SchemaException error) {
logger.error("failed to parse {} due to ", Namespace.class, error);
}
return namespace;
}
}

2
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/schemas/PropertyType.java
View File

@@ -50,8 +50,10 @@ public abstract class PropertyType {
@JsonProperty
private String apiType;
@JsonProperty(defaultValue = "true")
private boolean nullable;
@JsonProperty(required = true)
private TypeKind type;

192
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/io/RowReaderTest.java Normal file
View File

@@ -0,0 +1,192 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.io;
import com.azure.data.cosmos.core.Out;
import com.azure.data.cosmos.core.Utf8String;
import com.azure.data.cosmos.serialization.hybridrow.HybridRowVersion;
import com.azure.data.cosmos.serialization.hybridrow.Result;
import com.azure.data.cosmos.serialization.hybridrow.RowBuffer;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutCode;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutResolver;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutResolverNamespace;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutType;
import com.azure.data.cosmos.serialization.hybridrow.schemas.Namespace;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Factory;
import org.testng.annotations.Test;
import org.testng.util.Strings;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import static com.google.common.base.Strings.lenientFormat;
import static org.testng.Assert.*;
public class RowReaderTest {
private static final String basedir = System.getProperty("project.basedir", System.getProperty("user.dir"));
private final File schemaFile;
private final Path dataFile;
private Namespace namespace;
private RowReaderTest(File schemaFile, Path dataFile) {
this.schemaFile = schemaFile;
this.dataFile = dataFile;
}
@BeforeClass(groups = "unit")
public void setUp() {
assertNull(this.namespace);
this.namespace = Namespace.parse(this.schemaFile).orElseThrow(() ->
new AssertionError(lenientFormat("failed to load %s", this.schemaFile))
);
}
@Test(groups = "unit")
public void testRead() {
final long length = this.dataFile.toFile().length();
assertTrue(0 < length && length < Integer.MAX_VALUE);
final ByteBuf data = Unpooled.buffer((int) length);
try (InputStream stream = Files.newInputStream(this.dataFile)) {
data.writeBytes(stream, (int) length);
} catch (IOException error) {
fail(lenientFormat("failed to open %s due to %s", this.dataFile, error));
}
LayoutResolver resolver = new LayoutResolverNamespace(this.namespace);
RowBuffer buffer = new RowBuffer(data, HybridRowVersion.V1, resolver);
RowReader reader = new RowReader(buffer);
visitFields(reader, 0);
}
private static Result visitFields(RowReader reader, int level) {
while (reader.read()) {
final Utf8String path = reader.path();
LayoutType type = reader.type();
if (type == null) {
fail(lenientFormat("path: %s, type: null", path));
}
System.out.println(lenientFormat("%s%s:\"%s\"", Strings.repeat(" ", level), path, type.layoutCode()));
Out out = new Out();
switch (type.layoutCode()) {
case UINT_64: {
Result result = reader.readUInt64(out);
break;
}
case NULL:
case BOOLEAN:
case BOOLEAN_FALSE:
case INT_8:
case INT_16:
case INT_32:
case INT_64:
case UINT_8:
case UINT_16:
case UINT_32:
case VAR_INT:
case VAR_UINT:
case FLOAT_32:
case FLOAT_64:
case FLOAT_128:
case DECIMAL:
case DATE_TIME:
case UNIX_DATE_TIME:
case GUID:
case UTF_8:
case BINARY: {
break;
}
case NULLABLE_SCOPE:
case IMMUTABLE_NULLABLE_SCOPE: {
if (!reader.hasValue()) {
break;
}
}
case ARRAY_SCOPE:
case IMMUTABLE_ARRAY_SCOPE:
case MAP_SCOPE:
case IMMUTABLE_MAP_SCOPE:
case OBJECT_SCOPE:
case IMMUTABLE_OBJECT_SCOPE:
case SCHEMA:
case IMMUTABLE_SCHEMA:
case SET_SCOPE:
case IMMUTABLE_SET_SCOPE:
case TAGGED2_SCOPE:
case IMMUTABLE_TAGGED2_SCOPE:
case TAGGED_SCOPE:
case IMMUTABLE_TAGGED_SCOPE:
case TUPLE_SCOPE:
case IMMUTABLE_TUPLE_SCOPE:
case TYPED_ARRAY_SCOPE:
case IMMUTABLE_TYPED_ARRAY_SCOPE:
case TYPED_MAP_SCOPE:
case IMMUTABLE_TYPED_MAP_SCOPE:
case TYPED_SET_SCOPE:
case IMMUTABLE_TYPED_SET_SCOPE:
case TYPED_TUPLE_SCOPE:
case IMMUTABLE_TYPED_TUPLE_SCOPE: {
Result result = reader.readScope(null, (RowReader child, Object ignored) -> visitFields(child, level + 1));
if (result != Result.SUCCESS) {
return result;
}
break;
}
case END_SCOPE:
case INVALID:
case MONGODB_OBJECT_ID: {
fail(lenientFormat("unsupported layout type: %s", type));
break;
}
default: {
fail(lenientFormat("unknown layout type: %s", type));
break;
}
}
}
return Result.SUCCESS;
}
public static class Builder {
@Factory
public static Object[] create() {
return new Object[] {
new RowReaderTest(
Paths.get(basedir, "test-data", "RootSegment.json").toFile(),
Paths.get(basedir, "test-data", "RootSegment.hybridrow")
)
};
}
}
}

119
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/schemas/NamespaceTest.java
View File

@@ -4,46 +4,37 @@
package com.azure.data.cosmos.serialization.hybridrow.schemas;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Factory;
import org.testng.annotations.Test;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.io.File;
import java.nio.file.Paths;
import java.util.UUID;
import static com.google.common.base.Strings.lenientFormat;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertThrows;
import static org.testng.Assert.fail;
public class NamespaceTest {
private static final String basedir = System.getProperty("project.basedir", System.getProperty("user.dir"));
private final Path schemaPath;
private final File schemaFile;
private Namespace namespace;
private NamespaceTest(Path schemaPath) {
this.schemaPath = schemaPath;
private NamespaceTest(File schemaFile) {
this.schemaFile = schemaFile;
}
@BeforeClass(groups = "unit")
public void setUp() {
assertNull(this.namespace);
try (InputStream stream = Files.newInputStream(this.schemaPath)) {
this.namespace = Namespace.parse(stream).orElseThrow(() ->
new AssertionError(lenientFormat("failed to parse %s", this.schemaPath))
);
} catch (IOException error) {
fail(lenientFormat("unexpected %s", error));
}
this.namespace = Namespace.parse(this.schemaFile).orElseThrow(() ->
new AssertionError(lenientFormat("failed to load %s", this.schemaFile))
);
}
@Test(groups = "unit")
@@ -55,30 +46,98 @@ public class NamespaceTest {
@Test(groups = "unit")
public void testSchemas() {
}
@Test(groups = "unit")
public void testTestParse() {
}
assertNotNull(this.namespace.schemas());
@Test(groups = "unit")
public void testTestSchemas() {
}
for (Schema schema : this.namespace.schemas()) {
@Test(groups = "unit")
public void testTestVersion() {
assertNotNull(schema.name());
assertNotNull(schema.schemaId());
assertNotNull(schema.properties());
for (Property property : schema.properties()) {
assertNotNull(property.path());
assertNotNull(property.type());
assertValidPropertyType(property.type());
}
}
}
@Test(groups = "unit")
public void testVersion() {
assertNotNull(this.namespace.version());
assertThrows(NullPointerException.class, () -> this.namespace.version(null));
for (SchemaLanguageVersion version : SchemaLanguageVersion.values()) {
this.namespace.version(version);
assertEquals(this.namespace.version(), version);
}
}
private static void assertValidPropertyType(PropertyType propertyType) {
if (propertyType instanceof UdtPropertyType) {
UdtPropertyType value = (UdtPropertyType) propertyType;
assertNotNull(value.name());
assertNotNull(value.schemaId());
assertEquals(value.type(), TypeKind.SCHEMA);
return;
}
if (propertyType instanceof ArrayPropertyType) {
ArrayPropertyType value = (ArrayPropertyType) propertyType;
assertEquals(value.type(), TypeKind.ARRAY);
if (value.items() != null) {
assertValidPropertyType(value.items());
}
return;
}
if (propertyType instanceof PrimitivePropertyType) {
PrimitivePropertyType value = (PrimitivePropertyType) propertyType;
switch (value.type()) {
case BOOLEAN:
case NULL:
case INT_8:
case INT_16:
case INT_32:
case INT_64:
case VAR_INT:
case UINT_8:
case UINT_16:
case UINT_32:
case UINT_64:
case VAR_UINT:
case DECIMAL:
case FLOAT_32:
case FLOAT_64:
case FLOAT_128:
case GUID:
case DATE_TIME:
case UNIX_DATE_TIME:
case BINARY:
case UTF_8:
assertNotNull(value.storage());
assertNotEquals(value.storage(), StorageKind.NONE);
break;
default:
fail(lenientFormat("not a primitive TypeKind: %s", value.type()));
}
return;
}
fail(lenientFormat("untested property type: %s", propertyType));
}
public static class Builder {
@Factory
public static Object[] create() {
return new Object[] {
new NamespaceTest(Paths.get(basedir, "test-data", "RootSegment.json"))
new NamespaceTest(Paths.get(basedir, "test-data", "RootSegment.json").toFile())
};
}
}
}
}