random-mirrors/HybridRow
1
0
Fork 0
mirror of https://github.com/microsoft/HybridRow.git synced 2026-01-20 09:53:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added and debugged some tests

Browse Source
This commit is contained in:
David Noble
2019-09-14 18:43:04 -07:00
parent a25ea11aae
commit f1959ba5b3
5 changed files with 139 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DateTimeCodec.java
View File

@@ -37,6 +37,8 @@ import static com.google.common.base.Preconditions.checkNotNull;
* </td></tr>
* </tbody>
* </table>
*
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">struct DateTime source</a>
*/
public final class DateTimeCodec {
@@ -62,8 +64,7 @@ public final class DateTimeCodec {
*
* @param bytes a {@link byte} array containing the serialized value to be decoded.
* @return a new {@link OffsetDateTime}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">
* struct DateTimeCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">struct DateTime source</a>
*/
public static OffsetDateTime decode(@Nonnull final byte[] bytes) {
checkNotNull(bytes);
@@ -75,8 +76,7 @@ public final class DateTimeCodec {
*
* @param in a {@link ByteBuf} containing the serialized value to be decoded.
* @return a new {@link OffsetDateTime}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">
* struct DateTimeCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">struct DateTime source</a>
*/
public static OffsetDateTime decode(@Nonnull final ByteBuf in) {
@@ -101,8 +101,7 @@ public final class DateTimeCodec {
*
* @param offsetDateTime an {@link OffsetDateTime} to be encoded.
* @return a new byte array containing the encoded {@code offsetDateTime}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">
* struct DateTimeCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">struct DateTime source</a>
*/
public static byte[] encode(final OffsetDateTime offsetDateTime) {
final byte[] bytes = new byte[BYTES];
@@ -115,8 +114,7 @@ public final class DateTimeCodec {
*
* @param offsetDateTime an {@link OffsetDateTime} to be encoded.
* @param out an output {@link ByteBuf}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">
* struct DateTimeCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/a.html#df6b1eba7461813b">struct DateTime source</a>
*/
public static void encode(final OffsetDateTime offsetDateTime, final ByteBuf out) {

106
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DecimalCodec.java
View File

@@ -5,12 +5,14 @@ package com.azure.data.cosmos.serialization.hybridrow.codecs;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.util.ByteProcessor;
import javax.annotation.Nonnull;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.Arrays;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -18,6 +20,16 @@ import static com.google.common.base.Preconditions.checkState;
import static java.lang.Math.max;
import static java.lang.Math.min;
/**
* Provides static methods for encoding and decoding {@link BigDecimal}s serialized as {@code System.Decimal}s
*
* The serialization format is lossy as the {@link BigDecimal} class represents arbitrary-precision signed decimal
* numbers while the binary representation of a {@code System.Decimal} value is constrained to a magnitude of 96-bits
* with a scaling factor of 10 and a scale value between 0 and 28. This yields a precision between 28 and 29
* decimal digits.
*
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">struct Decimal source</a>
*/
public final class DecimalCodec {
public static final int BYTES = 4 * Integer.BYTES;
@@ -26,21 +38,26 @@ public final class DecimalCodec {
private static final int FLAGS_MASK_POWER = 0b00000000111111110000000000000000;
private static final int FLAGS_MASK_SIGN = 0b10000000000000000000000000000000;
private static final BigInteger MAGNITUDE_MAX = new BigInteger(new byte[] { (byte)0x00,
private static final BigInteger MAGNITUDE_MAX = new BigInteger(new byte[] {
(byte)0x00,
(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF,
(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF,
(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF });
private static final BigInteger MAGNITUDE_MIN = new BigInteger(new byte[] { (byte)0xFF,
private static final BigInteger MAGNITUDE_MIN = new BigInteger(new byte[] {
(byte)0xFF,
(byte)0x00, (byte)0x00, (byte)0x00, (byte)0x00,
(byte)0x00, (byte)0x00, (byte)0x00, (byte)0x00,
(byte)0x00, (byte)0x00, (byte)0x00, (byte)0x01 });
private static final MathContext REDUCED_PRECISION = new MathContext(28, RoundingMode.HALF_EVEN);
private static final int SCALE_MAX = 28;
private static final int SCALE_MIN = 0;
private static final int SCALE_SHIFT = 16;
private static final int VALUE_LENGTH = 3 * Integer.BYTES;
private static final int[] VALUE_OFFSETS = { /* hi */ 0, /* lo */ 2 * Integer.BYTES, /* mid */ Integer.BYTES };
private DecimalCodec() {
}
@@ -50,8 +67,7 @@ public final class DecimalCodec {
*
* @param bytes a {@link byte} array containing the serialized {@code System.Decimal} to be decoded.
* @return a new {@link BigDecimal}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">
* struct DecimalCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">struct Decimal source</a>
*/
public static BigDecimal decode(@Nonnull final byte[] bytes) {
checkNotNull(bytes);
@@ -63,8 +79,7 @@ public final class DecimalCodec {
*
* @param in a {@link ByteBuf} containing the serialized {@code System.Decimal} to be decoded.
* @return a new {@link BigDecimal}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">
* struct DecimalCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">struct Decimal source</a>
*/
public static BigDecimal decode(@Nonnull final ByteBuf in) {
@@ -74,35 +89,40 @@ public final class DecimalCodec {
BYTES,
in.readableBytes());
// The flags field is--and must be--interpreted as follows
// The flags field is interpreted as follows
//
// Bits Interpretation
// ----- --------------------------------------------------------------------------------------------
// 0-15 unused and must be zero
// 00-15 unused and must be zero
// 16-23 a value between 0 and 28 specifying the number of digits to the right of the decimal point
// 24-30 unused and must be zero
// 31 specifies the sign of the value, 1 meaning negative and 0 meaning non-negative
// 31-31 specifies the sign of the value, 1 meaning negative and 0 meaning non-negative
final int flags = in.readIntLE();
checkState((flags & FLAGS_MASK_INVALID) == 0, "invalid flags field: %s", flags);
final int scale = (flags | FLAGS_MASK_POWER) >>> SCALE_SHIFT;
final int scale = (flags & FLAGS_MASK_POWER) >>> SCALE_SHIFT;
checkState(scale <= SCALE_MAX);
final int signum = (flags & FLAGS_MASK_SIGN) == 0 ? 1 : -1;
final byte[] buffer = new byte[VALUE_LENGTH];
in.readBytes(buffer);
final byte[] magnitude = new byte[VALUE_LENGTH];
final int source = in.readerIndex();
int target = 0;
int i = VALUE_LENGTH;
int j = 0;
while (i > j) {
final byte b = buffer[--i];
buffer[i] = buffer[j];
buffer[j++] = b;
for (int offset : VALUE_OFFSETS) {
final int start = target;
in.forEachByteDesc(source + offset, Integer.BYTES, new ByteProcessor() {
int index = start;
@Override
public boolean process(byte value) {
magnitude[this.index++] = value;
return true;
}
});
target += Integer.BYTES;
}
return new BigDecimal(new BigInteger(signum, buffer), scale);
return new BigDecimal(new BigInteger(signum, magnitude), scale);
}
/**
@@ -110,13 +130,12 @@ public final class DecimalCodec {
*
* @param bigDecimal a {@link BigDecimal} to be encoded.
* @return a new byte array containing the encoded {@code bigDecimal}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">
* struct DecimalCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">struct Decimal source</a>
*/
public static byte[] encode(final BigDecimal bigDecimal) {
final byte[] bytes = new byte[BYTES];
encode(bigDecimal, Unpooled.wrappedBuffer(bytes));
return bytes;
ByteBuf buffer = Unpooled.wrappedBuffer(new byte[BYTES]).clear();
encode(bigDecimal, buffer);
return buffer.array();
}
/**
@@ -124,8 +143,7 @@ public final class DecimalCodec {
*
* @param value a {@link BigDecimal} to be encoded.
* @param out an output {@link ByteBuf}.
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">
* struct DecimalCodec source</a>
* @see <a href="https://referencesource.microsoft.com/mscorlib/system/decimal.cs.html">struct Decimal source</a>
*/
public static void encode(@Nonnull BigDecimal value, @Nonnull final ByteBuf out) {
@@ -158,30 +176,26 @@ public final class DecimalCodec {
unscaledValue = value.unscaledValue();
}
final byte[] bytes = unscaledValue.toByteArray();
final byte[] buffer = new byte[VALUE_LENGTH];
final byte[] decimalParts = new byte[VALUE_LENGTH];
final byte[] bytes;
final int flags;
if (signum == -1) {
flags = FLAGS_MASK_SIGN | value.scale() << SCALE_SHIFT;
int carry = 1;
for (int i = buffer.length, j = 0; i-- > 0; j++) {
final int n = ((bytes[i] ^ 0b1111_1111) & 0b1111_1111) + carry;
if ((n & 0b1_0000_0000) == 0) {
carry = 0;
}
buffer[j] = bytes[i];
}
} else {
if (signum > 0) {
flags = value.scale() << SCALE_SHIFT;
for (int i = buffer.length, j = 0; i-- > 0; j++) {
buffer[j] = bytes[i];
}
bytes = unscaledValue.toByteArray();
} else {
flags = value.scale() << SCALE_SHIFT | FLAGS_MASK_SIGN;
bytes = unscaledValue.abs().toByteArray();
}
out.writeIntLE(flags);
out.writeBytes(buffer, 0, 4);
out.writeBytes(buffer, 8, 4);
out.writeBytes(buffer, 4, 4);
for (int i = bytes.length, j = 0; i > 0 && j < decimalParts.length; ) {
decimalParts[j++] = bytes[--i];
}
out.writeBytes(decimalParts, 2 * Integer.BYTES, Integer.BYTES); // high
out.writeBytes(decimalParts, 0, Integer.BYTES); // low
out.writeBytes(decimalParts, Integer.BYTES, Integer.BYTES); // mid
}
}

65
java/src/main/java/com/azure/data/cosmos/serialization/hybridrow/codecs/GuidCodec.java
View File

@@ -12,6 +12,39 @@ import java.util.UUID;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Provides static methods for encoding and decoding {@link UUID}s serialized as {@code System.Guid}s
*
* {@link UUID}s are serialized like {@code System.Guid}s read and written by {@code MemoryMarshal.Read} and
* {@code MemoryMarshal.Write}.
*
* <table>
* <tbody>
* <tr><td>
* Bits 00-31
* </td><td>
* Contain an unsigned 32-bit int serialized in little-endian format.
* </td></tr>
* <tr><td>
* Bits 32-47
* </td><td>
* Contain an unsigned 16-bit int serialized in little-endian format.
* </td></tr>
* <tr><td>
* Bits 48-63
* </td><td>
* Contain an unsigned 16-bit int serialized in little-endian format.
* </td></tr>
* <tr><td>
* Bits 64-127
* </td><td>
* Contain an unsigned 64-bit int serialized in big-endian format.
* </td></tr>
* </tbody>
* </table>
*
* @see <a href="https://referencesource.microsoft.com/#mscorlib/system/guid.cs">struct Guid source</a>
*/
public final class GuidCodec {
public static final int BYTES = 2 * Long.BYTES;
@@ -34,29 +67,21 @@ public final class GuidCodec {
/**
* Decode a {@link UUID} serialized like a {@code System.Guid} by {@code MemoryMarshal.Write}.
*
* @param in a {@link ByteBuf} containing the serialized {@link UUID} to be decoded;
* @return a new {@link UUID}
* @param in a {@link ByteBuf} containing the serialized {@link UUID} to be decoded.
* @return a new {@link UUID}.
*/
public static UUID decode(@Nonnull final ByteBuf in) {
checkNotNull(in, "expected non-null in");
checkArgument(in.readableBytes() >= BYTES, "expected %s readable bytes, not %s",
BYTES,
in.readableBytes());
checkArgument(in.readableBytes() >= BYTES, "expected %s readable bytes, not %s bytes",
BYTES, in.readableBytes());
long mostSignificantBits = in.readUnsignedIntLE() << 32;
final long mostSignificantBits = in.readUnsignedIntLE() << Integer.SIZE
| (0x000000000000FFFFL & in.readShortLE()) << Short.SIZE
| (0x000000000000FFFFL & in.readShortLE());
mostSignificantBits |= (0x000000000000FFFFL & in.readShortLE()) << 16;
mostSignificantBits |= (0x000000000000FFFFL & in.readShortLE());
long leastSignificantBits = (0x000000000000FFFFL & in.readShortLE()) << (32 + 16);
for (int shift = 32 + 8; shift >= 0; shift -= 8) {
leastSignificantBits |= (0x00000000000000FFL & in.readByte()) << shift;
}
return new UUID(mostSignificantBits, leastSignificantBits);
return new UUID(mostSignificantBits, in.readLong());
}
/**
@@ -67,7 +92,7 @@ public final class GuidCodec {
*/
public static byte[] encode(final UUID uuid) {
final byte[] bytes = new byte[BYTES];
encode(uuid, Unpooled.wrappedBuffer(bytes));
encode(uuid, Unpooled.wrappedBuffer(bytes).clear());
return bytes;
}
@@ -85,10 +110,6 @@ public final class GuidCodec {
out.writeShortLE((short) ((mostSignificantBits & 0x00000000FFFF0000L) >>> 16));
out.writeShortLE((short) (mostSignificantBits & 0x000000000000FFFFL));
final long leastSignificantBits = uuid.getLeastSignificantBits();
out.writeShortLE((short) ((leastSignificantBits & 0xFFFF000000000000L) >>> (32 + 16)));
out.writeShort((short) ((leastSignificantBits & 0x0000FFFF00000000L) >>> 32));
out.writeInt((int) (leastSignificantBits & 0x00000000FFFFFFFFL));
out.writeLong(uuid.getLeastSignificantBits());
}
}

30
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/codecs/DecimalCodecTest.java
View File

@@ -19,10 +19,13 @@ import static org.testng.Assert.assertEquals;
* <p>
* Test data was generated from code that looks like this:
* {@code
* using System.Runtime.InteropServices;
* var buffer = new byte[16];
* var value = decimal.MaxValue;
* MemoryMarshal.Write(buffer, ref value);
* Console.WriteLine($"new DecimalItem(new byte[] {{ (byte) {string.Join(", (byte) ", buffer )} }}, new BigDecimal(\"{value.ToString()}\"))"); * }
* Console.WriteLine($"new DecimalItem(new byte[] {{ (byte) {string.Join(", (byte) ", buffer )} }}, new BigDecimal(\"
* {value.ToString()}\"))");
* }
*/
public class DecimalCodecTest {
@@ -34,7 +37,7 @@ public class DecimalCodecTest {
@Test(dataProvider = "decimalDataProvider")
public void testDecodeByteBuf(byte[] buffer, BigDecimal value) {
ByteBuf byteBuf = Unpooled.wrappedBuffer(new byte[DecimalCodec.BYTES]);
ByteBuf byteBuf = Unpooled.wrappedBuffer(buffer);
BigDecimal actual = DecimalCodec.decode(byteBuf);
assertEquals(actual, value);
}
@@ -47,14 +50,23 @@ public class DecimalCodecTest {
@Test(dataProvider = "decimalDataProvider")
public void testEncodeByteBuf(byte[] buffer, BigDecimal value) {
ByteBuf actual = Unpooled.wrappedBuffer(new byte[DecimalCodec.BYTES]);
ByteBuf actual = Unpooled.wrappedBuffer(new byte[DecimalCodec.BYTES]).clear();
DecimalCodec.encode(value, actual);
assertEquals(actual.array(), buffer);
}
@DataProvider(name = "decimalDataProvider")
private Iterator<Object[]> dateTimeData(byte[] buffer, BigDecimal value) {
private static Iterator<Object[]> decimalData() {
ImmutableList<DecimalItem> items = ImmutableList.of(
new DecimalItem( // decimal.MinusOne
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 128, // flags
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // high
(byte) 1, (byte) 0, (byte) 0, (byte) 0, // low
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // mid
},
new BigDecimal("-1")),
new DecimalItem( // decimal.Zero
new byte[] {
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // flags
@@ -86,8 +98,16 @@ public class DecimalCodecTest {
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // low
(byte) 255, (byte) 255, (byte) 255, (byte) 255, // mid
},
new BigDecimal("79228162514264337593543950335"))
new BigDecimal("79228162514264337593543950335")),
new DecimalItem( // new decimal(Math.PI)
new byte[] {
(byte) 0, (byte) 0, (byte) 14, (byte) 0, // flags
(byte) 0, (byte) 0, (byte) 0, (byte) 0, // high
(byte) 131, (byte) 36, (byte) 106, (byte) 231, // low
(byte) 185, (byte) 29, (byte) 1, (byte) 0 }, // mid
new BigDecimal("3.14159265358979"))
);
return items.stream().map(item -> new Object[] { item.buffer, item.value }).iterator();
}

8
java/src/test/java/com/azure/data/cosmos/serialization/hybridrow/codecs/GuidCodecTest.java
View File

@@ -35,7 +35,7 @@ public class GuidCodecTest {
@Test(dataProvider = "guidDataProvider")
public void testDecodeByteBuf(byte[] buffer, UUID value) {
ByteBuf byteBuf = Unpooled.wrappedBuffer(new byte[GuidCodec.BYTES]);
ByteBuf byteBuf = Unpooled.wrappedBuffer(buffer);
UUID actual = GuidCodec.decode(byteBuf);
assertEquals(actual, value);
}
@@ -48,13 +48,14 @@ public class GuidCodecTest {
@Test(dataProvider = "guidDataProvider")
public void testEncodeByteBuf(byte[] buffer, UUID value) {
ByteBuf actual = Unpooled.wrappedBuffer(new byte[GuidCodec.BYTES]);
ByteBuf actual = Unpooled.wrappedBuffer(new byte[GuidCodec.BYTES]).clear();
GuidCodec.encode(value, actual);
assertEquals(actual.array(), buffer);
}
@DataProvider(name = "guidDataProvider")
private Iterator<Object[]> guidData(byte[] buffer, UUID value) {
private static Iterator<Object[]> guidData() {
ImmutableList<GuidItem> items = ImmutableList.of(
new GuidItem(
new byte[] {
@@ -69,6 +70,7 @@ public class GuidCodecTest {
},
UUID.fromString("1bd611bf-aa16-4554-9369-c3d80151226b"))
);
return items.stream().map(item -> new Object[] { item.buffer, item.value }).iterator();
}