random-mirrors/HybridRow
1
0
Fork 0
mirror of https://github.com/microsoft/HybridRow.git synced 2026-01-19 09:23:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Progressed on port from dotnet to java

Browse Source
This commit is contained in:
David Noble
2019-09-03 14:21:38 -07:00
parent eb13d61895
commit 1445521237
12 changed files with 358 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
java/pom.xml
View File

@@ -44,6 +44,7 @@ Licensed under the MIT License.
<netty.version>4.1.39.Final</netty.version>
<protobuf.version>3.9.1</protobuf.version>
<slf4j.version>1.7.28</slf4j.version>
<testng.version>7.0.0</testng.version>
</properties>
<dependencies>
@@ -106,6 +107,13 @@ Licensed under the MIT License.
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>7.0.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>

85
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/RowBuffer.java
View File

@@ -6,10 +6,11 @@ package com.azure.data.cosmos.serialization.hybridrow;
import com.azure.data.cosmos.core.Out;
import com.azure.data.cosmos.core.Reference;
import com.azure.data.cosmos.core.Utf8String;
import com.azure.data.cosmos.core.codecs.DateTimeCodec;
import com.azure.data.cosmos.core.codecs.DecimalCodec;
import com.azure.data.cosmos.core.codecs.Float128Codec;
import com.azure.data.cosmos.core.codecs.GuidCodec;
import com.azure.data.cosmos.serialization.hybridrow.RowBuffer.UniqueIndexItem;
import com.azure.data.cosmos.serialization.hybridrow.codecs.DateTimeCodec;
import com.azure.data.cosmos.serialization.hybridrow.codecs.DecimalCodec;
import com.azure.data.cosmos.serialization.hybridrow.codecs.Float128Codec;
import com.azure.data.cosmos.serialization.hybridrow.codecs.GuidCodec;
import com.azure.data.cosmos.serialization.hybridrow.io.RowWriter;
import com.azure.data.cosmos.serialization.hybridrow.layouts.Layout;
import com.azure.data.cosmos.serialization.hybridrow.layouts.LayoutArray;
@@ -175,55 +176,63 @@ public final class RowBuffer {
return item.value();
}
public Float128 ReadFloat128(int offset) {
public int ReadInt32(int offset) {
Item<Integer> item = this.read(this.buffer::readIntLE, offset);
return item.value();
}
public Float128 readFloat128(int offset) {
Item<Float128> item = this.read(() -> Float128Codec.decode(this.buffer), offset);
return item.value();
}
public float ReadFloat32(int offset) {
return MemoryMarshal.<Float>Read(this.buffer.Slice(offset));
public float readFloat32(int offset) {
Item<Float> item = this.read(this.buffer::readFloatLE, offset);
return item.value();
}
public double ReadFloat64(int offset) {
return MemoryMarshal.<Double>Read(this.buffer.Slice(offset));
public double readFloat64(int offset) {
Item<Double> item = this.read(this.buffer::readDoubleLE, offset);
return item.value();
}
public int ReadInt32(int offset) {
return MemoryMarshal.<Integer>Read(this.buffer.Slice(offset));
// TODO: DANOBLE: resurrect MongoDbObjectId
// public MongoDbObjectId ReadMongoDbObjectId(int offset) {
// return MemoryMarshal.<MongoDbObjectId>Read(this.buffer.Slice(offset));
// }
public SchemaId readSchemaId(int offset) {
Item<SchemaId> item = this.read(() -> SchemaId.from(this.buffer.readIntLE()), offset);
return item.value();
}
public MongoDbObjectId ReadMongoDbObjectId(int offset) {
return MemoryMarshal.<MongoDbObjectId>Read(this.buffer.Slice(offset));
public BigDecimal readSparseDecimal(RowCursor edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutTypes.DECIMAL);
BigDecimal value = this.readDecimal(edit.valueOffset());
edit.endOffset(this.buffer.readerIndex());
return value;
}
public SchemaId ReadSchemaId(int offset) {
return new SchemaId(this.ReadInt32(offset));
public Float128 readSparseFloat128(RowCursor edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutTypes.FLOAT_128);
Float128 value = this.readFloat128(edit.valueOffset());
edit.endOffset(this.buffer.readerIndex());
return value;
}
public BigDecimal ReadSparseDecimal(Reference<RowCursor> edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutType.Decimal);
// TODO: C# TO JAVA CONVERTER: There is no Java equivalent to 'sizeof':
edit.get().endOffset = edit.get().valueOffset() + sizeof(BigDecimal);
return this.readDecimal(edit.get().valueOffset());
public float readSparseFloat32(RowCursor edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutTypes.FLOAT_32);
float value = this.readFloat32(edit.valueOffset());
edit.endOffset(this.buffer.readerIndex());
return value;
}
public Float128 ReadSparseFloat128(Reference<RowCursor> edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutType.Float128);
edit.get().endOffset = edit.get().valueOffset() + Float128.SIZE;
return this.ReadFloat128(edit.get().valueOffset()).clone();
}
public float ReadSparseFloat32(Reference<RowCursor> edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutType.Float32);
edit.get().endOffset = edit.get().valueOffset() + (Float.SIZE / Byte.SIZE);
return this.ReadFloat32(edit.get().valueOffset());
}
public MongoDbObjectId ReadSparseMongoDbObjectId(Reference<RowCursor> edit) {
this.readSparsePrimitiveTypeCode(edit, MongoDbObjectId);
edit.get().endOffset = edit.get().valueOffset() + MongoDbObjectId.Size;
return this.ReadMongoDbObjectId(edit.get().valueOffset()).clone();
}
// TODO: DANOBLE: resurrect MongoDbObjectId
// public MongoDbObjectId ReadSparseMongoDbObjectId(Reference<RowCursor> edit) {
// this.readSparsePrimitiveTypeCode(edit, MongoDbObjectId);
// edit.get().endOffset = edit.get().valueOffset() + MongoDbObjectId.Size;
// return this.ReadMongoDbObjectId(edit.get().valueOffset()).clone();
// }
public Utf8Span ReadSparseString(Reference<RowCursor> edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutType.Utf8);
@@ -1769,7 +1778,7 @@ public final class RowBuffer {
public double readSparseFloat64(RowCursor edit) {
this.readSparsePrimitiveTypeCode(edit, LayoutTypes.FLOAT_64);
edit.endOffset(edit.valueOffset() + (Double.SIZE / Byte.SIZE));
return this.ReadFloat64(edit.valueOffset());
return this.readFloat64(edit.valueOffset());
}
public UUID readSparseGuid(RowCursor edit) {

4
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/RowCursor.java
View File

@@ -50,7 +50,7 @@ public final class RowCursor implements Cloneable {
public static RowCursor Create(RowBuffer row) {
final SchemaId schemaId = row.ReadSchemaId(1);
final SchemaId schemaId = row.readSchemaId(1);
final Layout layout = row.resolver().resolve(schemaId);
final int sparseSegmentOffset = row.computeVariableValueOffset(layout, HybridRowHeader.SIZE, layout.numVariable());
@@ -65,7 +65,7 @@ public final class RowCursor implements Cloneable {
public static RowCursor CreateForAppend(RowBuffer row) {
final SchemaId schemaId = row.ReadSchemaId(1);
final SchemaId schemaId = row.readSchemaId(1);
final Layout layout = row.resolver().resolve(schemaId);
return new RowCursor()

4
java/src/main/java/com/azure/data/cosmos/core/codecs/DateTimeCodec.java → java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DateTimeCodec.java
View File

@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.core.codecs;
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
@@ -16,7 +16,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
public final class DateTimeCodec {
private static final int BYTES = Long.BYTES;
public static final int BYTES = Long.BYTES;
private static final long FLAGS_MASK = 0xC000000000000000L;
private static final long KIND_AMBIGUOUS = 0xC000000000000000L;

2
java/src/main/java/com/azure/data/cosmos/core/codecs/DecimalCodec.java → java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DecimalCodec.java
View File

@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.core.codecs;
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;

2
java/src/main/java/com/azure/data/cosmos/core/codecs/Float128Codec.java → java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/Float128Codec.java
View File

@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.core.codecs;
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import com.azure.data.cosmos.serialization.hybridrow.Float128;
import io.netty.buffer.ByteBuf;

2
java/src/main/java/com/azure/data/cosmos/core/codecs/GuidCodec.java → java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/GuidCodec.java
View File

@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.core.codecs;
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;

6
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/io/RowReader.java
View File

@@ -453,7 +453,7 @@ public final class RowReader {
Reference<RowCursor> tempReference_cursor =
new Reference<RowCursor>(this.cursor);
value.setAndGet(this.row.ReadSparseDecimal(tempReference_cursor));
value.setAndGet(this.row.readSparseDecimal(tempReference_cursor));
this.cursor = tempReference_cursor.get();
return Result.Success;
default:
@@ -480,7 +480,7 @@ public final class RowReader {
Reference<RowCursor> tempReference_cursor =
new Reference<RowCursor>(this.cursor);
value.setAndGet(this.row.ReadSparseFloat128(tempReference_cursor).clone());
value.setAndGet(this.row.readSparseFloat128(tempReference_cursor).clone());
this.cursor = tempReference_cursor.get();
return Result.Success;
default:
@@ -507,7 +507,7 @@ public final class RowReader {
Reference<RowCursor> tempReference_cursor =
new Reference<RowCursor>(this.cursor);
value.setAndGet(this.row.ReadSparseFloat32(tempReference_cursor));
value.setAndGet(this.row.readSparseFloat32(tempReference_cursor));
this.cursor = tempReference_cursor.get();
return Result.Success;
default:

2
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/layouts/LayoutUDT.java
View File

@@ -27,7 +27,7 @@ public final class LayoutUDT extends LayoutPropertyScope {
@Override
public TypeArgumentList readTypeArgumentList(Reference<RowBuffer> row, int offset,
Out<Integer> lenInBytes) {
SchemaId schemaId = row.get().ReadSchemaId(offset).clone();
SchemaId schemaId = row.get().readSchemaId(offset).clone();
lenInBytes.setAndGet(SchemaId.SIZE);
return new TypeArgumentList(schemaId.clone());
}

87
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DateTimeCodecTest.java Normal file
View File

@@ -0,0 +1,87 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import com.google.common.collect.ImmutableList;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.time.OffsetDateTime;
import java.util.Iterator;
import static org.testng.Assert.assertEquals;
/**
* Tests the DateTimeCodec using data generated from C# code
* <p>
* Test data was generated from code that looks like this:
* {@code
* var buffer = new byte[8];
* var value = DateTime.Now;
* MemoryMarshal.Write(buffer, ref value);
* Console.WriteLine($"new DateTimeItem(new byte[] {{ (byte) {string.Join(", (byte) ", buffer )} }}, OffsetDateTime.parse(\"{value.ToString("o")}\"))");
* }
*/
public class DateTimeCodecTest {
@Test(dataProvider = "dateTimeDataProvider")
public void testDecodeByteArray(byte[] buffer, OffsetDateTime value) {
OffsetDateTime actual = DateTimeCodec.decode(buffer);
assertEquals(actual, value);
}
@Test(dataProvider = "dateTimeDataProvider")
public void testDecodeByteBuf(byte[] buffer, OffsetDateTime value) {
ByteBuf byteBuf = Unpooled.wrappedBuffer(new byte[DateTimeCodec.BYTES]);
OffsetDateTime actual = DateTimeCodec.decode(byteBuf);
assertEquals(actual, value);
}
@Test(dataProvider = "dateTimeDataProvider")
public void testEncodeByteArray(byte[] buffer, OffsetDateTime value) {
byte[] actual = DateTimeCodec.encode(value);
assertEquals(actual, buffer);
}
@Test(dataProvider = "dateTimeDataProvider")
public void testEncodeByteBuf(byte[] buffer, OffsetDateTime value) {
ByteBuf actual = Unpooled.wrappedBuffer(new byte[DateTimeCodec.BYTES]);
DateTimeCodec.encode(value, actual);
assertEquals(actual.array(), buffer);
}
@DataProvider(name = "dateTimeDataProvider")
private Iterator<Object[]> dateTimeData(byte[] buffer, OffsetDateTime value) {
ImmutableList<DateTimeItem> items = ImmutableList.of(
new DateTimeItem(new byte[] {
(byte) 120, (byte) 212, (byte) 106, (byte) 251, (byte) 105, (byte) 48, (byte) 215, (byte) 136 },
OffsetDateTime.parse("2019-09-03T12:26:44.3996280-07:00")),
new DateTimeItem(new byte[] {
(byte) 226, (byte) 108, (byte) 87, (byte) 194, (byte) 164, (byte) 48, (byte) 215, (byte) 72 },
OffsetDateTime.parse("2019-09-03T19:27:28.9493730Z"))
);
return items.stream().map(item -> new Object[] { item.buffer, item.value }).iterator();
}
private static class DateTimeItem {
private final byte[] buffer;
private final OffsetDateTime value;
DateTimeItem(byte[] buffer, OffsetDateTime value) {
this.buffer = buffer;
this.value = value;
}
public byte[] buffer() {
return this.buffer;
}
public OffsetDateTime value() {
return this.value;
}
}
}

112
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DecimalCodecTest.java Normal file
View File

@@ -0,0 +1,112 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import com.google.common.collect.ImmutableList;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.math.BigDecimal;
import java.util.Iterator;
import static org.testng.Assert.assertEquals;
/**
* Tests the DecimalCodec using data generated from C# code
* <p>
* Test data was generated from code that looks like this:
* {@code
* var buffer = new byte[16];
* var value = decimal.MaxValue;
* MemoryMarshal.Write(buffer, ref value);
* Console.WriteLine($"new DecimalItem(new byte[] {{ (byte) {string.Join(", (byte) ", buffer )} }}, new BigDecimal(\"{value.ToString()}\"))"); * }
*/
public class DecimalCodecTest {
@Test(dataProvider = "decimalDataProvider")
public void testDecodeByteArray(byte[] buffer, BigDecimal value) {
BigDecimal actual = DecimalCodec.decode(buffer);
assertEquals(actual, value);
}
@Test(dataProvider = "decimalDataProvider")
public void testDecodeByteBuf(byte[] buffer, BigDecimal value) {
ByteBuf byteBuf = Unpooled.wrappedBuffer(new byte[DecimalCodec.BYTES]);
BigDecimal actual = DecimalCodec.decode(byteBuf);
assertEquals(actual, value);
}
@Test(dataProvider = "decimalDataProvider")
public void testEncodeByteArray(byte[] buffer, BigDecimal value) {
byte[] actual = DecimalCodec.encode(value);
assertEquals(actual, buffer);
}
@Test(dataProvider = "decimalDataProvider")
public void testEncodeByteBuf(byte[] buffer, BigDecimal value) {
ByteBuf actual = Unpooled.wrappedBuffer(new byte[DecimalCodec.BYTES]);
DecimalCodec.encode(value, actual);
assertEquals(actual.array(), buffer);
}
@DataProvider(name = "decimalDataProvider")
private Iterator<Object[]> dateTimeData(byte[] buffer, BigDecimal value) {
ImmutableList<DecimalItem> items = ImmutableList.of(
new DecimalItem( // decimal.Zero
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // flags
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // high
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // low
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // mid
},
new BigDecimal("0")),
new DecimalItem( // decimal.One
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // flags
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // high
(byte) 1, (byte) 0, (byte) 0, (byte) 0, // low
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // mid
},
new BigDecimal("1")),
new DecimalItem( // decimal.MinValue
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 128, // flags
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // high
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // low
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // mid
},
new BigDecimal("-79228162514264337593543950335")),
new DecimalItem( // decimal.MaxValue
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // flags
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // high
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // low
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // mid
},
new BigDecimal("79228162514264337593543950335"))
);
return items.stream().map(item -> new Object[] { item.buffer, item.value }).iterator();
}
private static class DecimalItem {
private final byte[] buffer;
private final BigDecimal value;
DecimalItem(byte[] buffer, BigDecimal value) {
this.buffer = buffer;
this.value = value;
}
public byte[] buffer() {
return this.buffer;
}
public BigDecimal value() {
return this.value;
}
}
}

93
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/codecs/GuidCodecTest.java Normal file
View File

@@ -0,0 +1,93 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.data.cosmos.serialization.hybridrow.codecs;
import com.google.common.collect.ImmutableList;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.Iterator;
import java.util.UUID;
import static org.testng.Assert.assertEquals;
/**
* Tests the GuidCodec using data generated from C# code
* <p>
* Test data was generated from code that looks like this:
* {@code
* var buffer = new byte[16];
* var value = Guid.NewGuid();
* MemoryMarshal.Write(buffer, ref value);
* Console.WriteLine($"new GuidItem(new byte[] {{ (byte) {string.Join(", (byte) ", buffer )} }}, UUID.fromString(\"{value.ToString()}\"))");
* }
*/
public class GuidCodecTest {
@Test(dataProvider = "guidDataProvider")
public void testDecodeByteArray(byte[] buffer, UUID value) {
UUID actual = GuidCodec.decode(buffer);
assertEquals(actual, value);
}
@Test(dataProvider = "guidDataProvider")
public void testDecodeByteBuf(byte[] buffer, UUID value) {
ByteBuf byteBuf = Unpooled.wrappedBuffer(new byte[GuidCodec.BYTES]);
UUID actual = GuidCodec.decode(byteBuf);
assertEquals(actual, value);
}
@Test(dataProvider = "guidDataProvider")
public void testEncodeByteArray(byte[] buffer, UUID value) {
byte[] actual = GuidCodec.encode(value);
assertEquals(actual, buffer);
}
@Test(dataProvider = "guidDataProvider")
public void testEncodeByteBuf(byte[] buffer, UUID value) {
ByteBuf actual = Unpooled.wrappedBuffer(new byte[GuidCodec.BYTES]);
GuidCodec.encode(value, actual);
assertEquals(actual.array(), buffer);
}
@DataProvider(name = "guidDataProvider")
private Iterator<Object[]> guidData(byte[] buffer, UUID value) {
ImmutableList<GuidItem> items = ImmutableList.of(
new GuidItem(
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0
},
UUID.fromString("00000000-0000-0000-0000-000000000000")),
new GuidItem(
new byte[] {
(byte) 191, (byte) 17, (byte) 214, (byte) 27, (byte) 22, (byte) 170, (byte) 84, (byte) 69,
(byte) 147, (byte) 105, (byte) 195, (byte) 216, (byte) 1, (byte) 81, (byte) 34, (byte) 107
},
UUID.fromString("1bd611bf-aa16-4554-9369-c3d80151226b"))
);
return items.stream().map(item -> new Object[] { item.buffer, item.value }).iterator();
}
private static class GuidItem {
private final byte[] buffer;
private final UUID value;
GuidItem(byte[] buffer, UUID value) {
this.buffer = buffer;
this.value = value;
}
public byte[] buffer() {
return this.buffer;
}
public UUID value() {
return this.value;
}
}
}