diff --git a/Hybrid Row Whitepaper.docx b/Hybrid Row Whitepaper.docx new file mode 100644 index 0000000..9635cd4 Binary files /dev/null and b/Hybrid Row Whitepaper.docx differ diff --git a/dotnet/src/HybridRow.Json/Microsoft.Azure.Cosmos.Serialization.HybridRow.Json.csproj b/dotnet/src/HybridRow.Json/Microsoft.Azure.Cosmos.Serialization.HybridRow.Json.csproj new file mode 100644 index 0000000..d4b4d50 --- /dev/null +++ b/dotnet/src/HybridRow.Json/Microsoft.Azure.Cosmos.Serialization.HybridRow.Json.csproj @@ -0,0 +1,21 @@ + + + + true + {CE1C4987-FC19-4887-9EB6-13508F8DA644} + Library + Microsoft.Azure.Cosmos.Serialization.HybridRow.Json + Microsoft.Azure.Cosmos.Serialization.HybridRow.Json + netstandard2.0 + AnyCPU + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/HybridRow.Json/Properties/AssemblyInfo.cs b/dotnet/src/HybridRow.Json/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..be43fcc --- /dev/null +++ b/dotnet/src/HybridRow.Json/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRow.Json")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("CE1C4987-FC19-4887-9EB6-13508F8DA644")] diff --git a/dotnet/src/HybridRow.Json/RowReaderJsonExtensions.cs b/dotnet/src/HybridRow.Json/RowReaderJsonExtensions.cs new file mode 100644 index 0000000..3341c9b --- /dev/null +++ b/dotnet/src/HybridRow.Json/RowReaderJsonExtensions.cs @@ -0,0 +1,453 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Json +{ + using System; + using System.Text; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public static class RowReaderJsonExtensions + { + /// + /// Project a JSON document from a HybridRow . + /// + /// The reader to project to JSON. + /// If successful, the JSON document that corresponds to the . + /// The result. + public static Result ToJson(this ref RowReader reader, out string str) + { + return reader.ToJson(new RowReaderJsonSettings(" "), out str); + } + + /// + /// Project a JSON document from a HybridRow . + /// + /// The reader to project to JSON. + /// Settings that control how the JSON document is formatted. + /// If successful, the JSON document that corresponds to the . + /// The result. + public static Result ToJson(this ref RowReader reader, RowReaderJsonSettings settings, out string str) + { + ReaderStringContext ctx = new ReaderStringContext( + new StringBuilder(), + new RowReaderJsonSettings(settings.IndentChars, settings.QuoteChar == '\'' ? '\'' : '"'), + 1); + + ctx.Builder.Append("{"); + Result result = RowReaderJsonExtensions.ToJson(ref reader, ctx); + if (result != Result.Success) + { + str = null; + return result; + } + + ctx.Builder.Append(ctx.NewLine); + ctx.Builder.Append("}"); + str = ctx.Builder.ToString(); + return Result.Success; + } + + private static Result ToJson(ref RowReader reader, ReaderStringContext ctx) + { + int index = 0; + while (reader.Read()) + { + string path = !reader.Path.IsNull ? $"{ctx.Settings.QuoteChar}{reader.Path}{ctx.Settings.QuoteChar}:" : null; + if (index != 0) + { + ctx.Builder.Append(','); + } + + index++; + ctx.Builder.Append(ctx.NewLine); + ctx.WriteIndent(); + if (path != null) + { + ctx.Builder.Append(path); + ctx.Builder.Append(ctx.Separator); + } + + Result r; + char scopeBracket = default; + char scopeCloseBracket = default; + switch (reader.Type.LayoutCode) + { + case LayoutCode.Null: + { + r = reader.ReadNull(out NullValue _); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append("null"); + break; + } + + case LayoutCode.Boolean: + { + r = reader.ReadBool(out bool value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Int8: + { + r = reader.ReadInt8(out sbyte value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Int16: + { + r = reader.ReadInt16(out short value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Int32: + { + r = reader.ReadInt32(out int value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Int64: + { + r = reader.ReadInt64(out long value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.UInt8: + { + r = reader.ReadUInt8(out byte value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.UInt16: + { + r = reader.ReadUInt16(out ushort value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.UInt32: + { + r = reader.ReadUInt32(out uint value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.UInt64: + { + r = reader.ReadUInt64(out ulong value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.VarInt: + { + r = reader.ReadVarInt(out long value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.VarUInt: + { + r = reader.ReadVarUInt(out ulong value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Float32: + { + r = reader.ReadFloat32(out float value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Float64: + { + r = reader.ReadFloat64(out double value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.Float128: + { + r = reader.ReadFloat128(out Float128 _); + if (r != Result.Success) + { + return r; + } + + // ctx.Builder.AppendFormat("High: {0}, Low: {1}\n", value.High, value.Low); + Contract.Assert(false, "Float128 are not supported."); + break; + } + + case LayoutCode.Decimal: + { + r = reader.ReadDecimal(out decimal value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + break; + } + + case LayoutCode.DateTime: + { + r = reader.ReadDateTime(out DateTime value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(ctx.Settings.QuoteChar); + ctx.Builder.Append(value); + ctx.Builder.Append(ctx.Settings.QuoteChar); + break; + } + + case LayoutCode.UnixDateTime: + { + r = reader.ReadUnixDateTime(out UnixDateTime value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value.Milliseconds); + break; + } + + case LayoutCode.Guid: + { + r = reader.ReadGuid(out Guid value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(ctx.Settings.QuoteChar); + ctx.Builder.Append(value.ToString()); + ctx.Builder.Append(ctx.Settings.QuoteChar); + break; + } + + case LayoutCode.MongoDbObjectId: + { + r = reader.ReadMongoDbObjectId(out MongoDbObjectId value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(ctx.Settings.QuoteChar); + ReadOnlyMemory bytes = value.ToByteArray(); + ctx.Builder.Append(bytes.Span.ToHexString()); + ctx.Builder.Append(ctx.Settings.QuoteChar); + break; + } + + case LayoutCode.Utf8: + { + r = reader.ReadString(out Utf8Span value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(ctx.Settings.QuoteChar); + ctx.Builder.Append(value.ToString()); + ctx.Builder.Append(ctx.Settings.QuoteChar); + break; + } + + case LayoutCode.Binary: + { + r = reader.ReadBinary(out ReadOnlySpan value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(ctx.Settings.QuoteChar); + ctx.Builder.Append(value.ToHexString()); + ctx.Builder.Append(ctx.Settings.QuoteChar); + break; + } + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + { + if (!reader.HasValue) + { + ctx.Builder.Append("null"); + break; + } + + goto case LayoutCode.TypedTupleScope; + } + + case LayoutCode.ArrayScope: + case LayoutCode.ImmutableArrayScope: + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + scopeBracket = '['; + scopeCloseBracket = ']'; + goto case LayoutCode.EndScope; + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + scopeBracket = '{'; + scopeCloseBracket = '}'; + goto case LayoutCode.EndScope; + + case LayoutCode.EndScope: + { + ctx.Builder.Append(scopeBracket); + int snapshot = ctx.Builder.Length; + r = reader.ReadScope(new ReaderStringContext(ctx.Builder, ctx.Settings, ctx.Indent + 1), RowReaderJsonExtensions.ToJson); + if (r != Result.Success) + { + return r; + } + + if (ctx.Builder.Length != snapshot) + { + ctx.Builder.Append(ctx.NewLine); + ctx.WriteIndent(); + } + + ctx.Builder.Append(scopeCloseBracket); + break; + } + + default: + { + Contract.Assert(false, $"Unknown type will be ignored: {reader.Type.LayoutCode}"); + break; + } + } + } + + return Result.Success; + } + + private readonly struct ReaderStringContext + { + public readonly int Indent; + public readonly StringBuilder Builder; + public readonly RowReaderJsonSettings Settings; + public readonly string Separator; + public readonly string NewLine; + + public ReaderStringContext(StringBuilder builder, RowReaderJsonSettings settings, int indent) + { + this.Settings = settings; + this.Separator = settings.IndentChars == null ? "" : " "; + this.NewLine = settings.IndentChars == null ? "" : "\n"; + this.Indent = indent; + this.Builder = builder; + } + + public void WriteIndent() + { + string indentChars = this.Settings.IndentChars ?? ""; + for (int i = 0; i < this.Indent; i++) + { + this.Builder.Append(indentChars); + } + } + } + } +} diff --git a/dotnet/src/HybridRow.Json/RowReaderJsonSettings.cs b/dotnet/src/HybridRow.Json/RowReaderJsonSettings.cs new file mode 100644 index 0000000..4955da9 --- /dev/null +++ b/dotnet/src/HybridRow.Json/RowReaderJsonSettings.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1051 // Do not declare visible instance fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Json +{ + public readonly struct RowReaderJsonSettings + { + /// If non-null then child objects are indented by one copy of this string per level. + public readonly string IndentChars; + + /// The quote character to use. + /// May be or . + public readonly char QuoteChar; + + public RowReaderJsonSettings(string indentChars = " ", char quoteChar = '"') + { + this.IndentChars = indentChars; + this.QuoteChar = quoteChar; + } + } +} diff --git a/dotnet/src/HybridRow.Package/Microsoft.Azure.Cosmos.Serialization.HybridRow.Package.nuproj b/dotnet/src/HybridRow.Package/Microsoft.Azure.Cosmos.Serialization.HybridRow.Package.nuproj new file mode 100644 index 0000000..91fda2c --- /dev/null +++ b/dotnet/src/HybridRow.Package/Microsoft.Azure.Cosmos.Serialization.HybridRow.Package.nuproj @@ -0,0 +1,26 @@ + + + + {7047DD2A-14FA-445E-93B2-67EB98282C4D} + External + true + + + + $(OutputRootDir) + $(NugetPackProperties);ProductSemanticVersion=$(ProductSemanticVersion);VersionPrereleaseExtension=$(VersionPrereleaseExtension) + + + + false + {B7621117-AEF3-4E10-928D-533AE893F379} + Microsoft.Azure.Cosmos.Core + + + false + {490D42EE-1FEF-47CC-97E4-782A353B4D58} + Microsoft.Azure.Cosmos.Serialization.HybridRow + + + + diff --git a/dotnet/src/HybridRow.Package/Microsoft.Azure.Cosmos.Serialization.HybridRow.Package.nuspec b/dotnet/src/HybridRow.Package/Microsoft.Azure.Cosmos.Serialization.HybridRow.Package.nuspec new file mode 100644 index 0000000..b557e87 --- /dev/null +++ b/dotnet/src/HybridRow.Package/Microsoft.Azure.Cosmos.Serialization.HybridRow.Package.nuspec @@ -0,0 +1,29 @@ + + + + Microsoft.Azure.Cosmos.Serialization.HybridRow + 1.0.0-preview + Microsoft.Azure.Cosmos.Serialization.HybridRow + Microsoft + Microsoft + https://aka.ms/netcoregaeula + https://github.com/Azure/azure-cosmos-dotnet-v3 + http://go.microsoft.com/fwlink/?LinkID=288890 + false + Microsoft.Azure.Cosmos.Serialization.HybridRow + This package supports the Microsoft Azure Cosmos Client and is not intended to be used directly from your code. + Copyright © Microsoft Corporation + microsoft azure cosmos dotnetcore netcore netstandard + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/HybridRow.Tests.Perf/App.config b/dotnet/src/HybridRow.Tests.Perf/App.config new file mode 100644 index 0000000..02249c3 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/App.config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/dotnet/src/HybridRow.Tests.Perf/BenchmarkSuiteBase.cs b/dotnet/src/HybridRow.Tests.Perf/BenchmarkSuiteBase.cs new file mode 100644 index 0000000..ba2740d --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/BenchmarkSuiteBase.cs @@ -0,0 +1,119 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + public class BenchmarkSuiteBase + { + private protected const int InitialCapacity = 2 * 1024 * 1024; + private protected LayoutResolverNamespace DefaultResolver = (LayoutResolverNamespace)SystemSchema.LayoutResolver; + + private protected async Task<(List>, LayoutResolverNamespace)> LoadExpectedAsync(string expectedFile) + { + LayoutResolverNamespace resolver = this.DefaultResolver; + List> expected = new List>(); + using (Stream stm = new FileStream(expectedFile, FileMode.Open)) + { + // Read a RecordIO stream. + MemorySpanResizer resizer = new MemorySpanResizer(BenchmarkSuiteBase.InitialCapacity); + Result r = await stm.ReadRecordIOAsync( + record => + { + r = BenchmarkSuiteBase.LoadOneRow(record, resolver, out Dictionary rowValue); + ResultAssert.IsSuccess(r); + expected.Add(rowValue); + return Result.Success; + }, + segment => + { + r = SegmentSerializer.Read(segment.Span, SystemSchema.LayoutResolver, out Segment s); + ResultAssert.IsSuccess(r); + Assert.IsNotNull(s.SDL); + resolver = new LayoutResolverNamespace(Namespace.Parse(s.SDL), resolver); + return Result.Success; + }, + resizer); + + ResultAssert.IsSuccess(r); + } + + return (expected, resolver); + } + + private protected static async Task WriteAllRowsAsync( + string file, + string sdl, + LayoutResolver resolver, + Layout layout, + List> rows) + { + using (Stream stm = new FileStream(file, FileMode.Truncate)) + { + // Create a reusable, resizable buffer. + MemorySpanResizer resizer = new MemorySpanResizer(BenchmarkSuiteBase.InitialCapacity); + + // Write a RecordIO stream. + Result r = await stm.WriteRecordIOAsync( + new Segment("HybridRow.Tests.Perf Expected Results", sdl), + (long index, out ReadOnlyMemory body) => + { + body = default; + if (index >= rows.Count) + { + return Result.Success; + } + + StreamingRowGenerator writer = new StreamingRowGenerator( + BenchmarkSuiteBase.InitialCapacity, + layout, + resolver, + resizer); + + Result r2 = writer.WriteBuffer(rows[(int)index]); + if (r2 != Result.Success) + { + return r2; + } + + body = resizer.Memory.Slice(0, writer.Length); + return Result.Success; + }); + + ResultAssert.IsSuccess(r); + } + } + + private protected static Result LoadOneRow(Memory buffer, LayoutResolver resolver, out Dictionary rowValue) + { + RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, resolver); + RowReader reader = new RowReader(ref row); + return DiagnosticConverter.ReaderToDynamic(ref reader, out rowValue); + } + + private protected static class ResultAssert + { + public static void IsSuccess(Result actual) + { + if (actual != Result.Success) + { + Assert.AreEqual(Result.Success, actual); + } + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/BsonJsonModelRowGenerator.cs b/dotnet/src/HybridRow.Tests.Perf/BsonJsonModelRowGenerator.cs new file mode 100644 index 0000000..8342aa9 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/BsonJsonModelRowGenerator.cs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using MongoDB.Bson.IO; + + internal sealed class BsonJsonModelRowGenerator : IDisposable + { + private readonly MemoryStream stream; + private readonly BsonWriter writer; + + public BsonJsonModelRowGenerator(int capacity) + { + this.stream = new MemoryStream(capacity); + this.writer = new BsonBinaryWriter(this.stream); + } + + public int Length => (int)this.stream.Position; + + public byte[] ToArray() + { + return this.stream.ToArray(); + } + + public void Reset() + { + this.stream.SetLength(0); + this.stream.Position = 0; + } + + public void WriteBuffer(Dictionary dict) + { + this.writer.WriteStartDocument(); + foreach ((Utf8String propPath, object propValue) in dict) + { + this.JsonModelSwitch(propPath, propValue); + } + + this.writer.WriteEndDocument(); + } + + public void Dispose() + { + this.writer.Dispose(); + this.stream.Dispose(); + } + + private void JsonModelSwitch(Utf8String path, object value) + { + if (path != null) + { + this.writer.WriteName(path.ToString()); + } + + switch (value) + { + case null: + this.writer.WriteNull(); + return; + case bool x: + this.writer.WriteBoolean(x); + return; + case long x: + this.writer.WriteInt64(x); + return; + case double x: + this.writer.WriteDouble(x); + return; + case string x: + this.writer.WriteString(x); + return; + case Utf8String x: + this.writer.WriteString(x.ToString()); + return; + case byte[] x: + this.writer.WriteBytes(x); + return; + case Dictionary x: + this.writer.WriteStartDocument(); + foreach ((Utf8String propPath, object propValue) in x) + { + this.JsonModelSwitch(propPath, propValue); + } + + this.writer.WriteEndDocument(); + return; + case List x: + this.writer.WriteStartArray(); + foreach (object item in x) + { + this.JsonModelSwitch(null, item); + } + + this.writer.WriteEndArray(); + + return; + default: + Contract.Assert(false, $"Unknown type will be ignored: {value.GetType().Name}"); + return; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/BsonReaderExtensions.cs b/dotnet/src/HybridRow.Tests.Perf/BsonReaderExtensions.cs new file mode 100644 index 0000000..c0b50d0 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/BsonReaderExtensions.cs @@ -0,0 +1,63 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using MongoDB.Bson; + using MongoDB.Bson.IO; + + internal static class BsonReaderExtensions + { + public static void VisitBsonDocument(this BsonReader bsonReader) + { + bsonReader.ReadStartDocument(); + BsonType type; + while ((type = bsonReader.ReadBsonType()) != BsonType.EndOfDocument) + { + string path = bsonReader.ReadName(); + switch (type) + { + case BsonType.Array: + bsonReader.VisitBsonArray(); + break; + + case BsonType.Document: + bsonReader.VisitBsonDocument(); + break; + + default: + bsonReader.SkipValue(); + break; + } + } + + bsonReader.ReadEndDocument(); + } + + private static void VisitBsonArray(this BsonReader bsonReader) + { + bsonReader.ReadStartArray(); + BsonType type; + while ((type = bsonReader.ReadBsonType()) != BsonType.EndOfDocument) + { + switch (type) + { + case BsonType.Array: + bsonReader.VisitBsonArray(); + break; + + case BsonType.Document: + bsonReader.VisitBsonDocument(); + break; + + default: + bsonReader.SkipValue(); + break; + } + } + + bsonReader.ReadEndArray(); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/BsonRowGenerator.cs b/dotnet/src/HybridRow.Tests.Perf/BsonRowGenerator.cs new file mode 100644 index 0000000..b5bdd68 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/BsonRowGenerator.cs @@ -0,0 +1,303 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using MongoDB.Bson; + using MongoDB.Bson.IO; + + internal sealed class BsonRowGenerator : IDisposable + { + private readonly MemoryStream stream; + private readonly BsonWriter writer; + private readonly Layout layout; + private readonly LayoutResolver resolver; + + public BsonRowGenerator(int capacity, Layout layout, LayoutResolver resolver) + { + this.stream = new MemoryStream(capacity); + this.writer = new BsonBinaryWriter(this.stream); + this.layout = layout; + this.resolver = resolver; + } + + public int Length => (int)this.stream.Position; + + public byte[] ToArray() + { + return this.stream.ToArray(); + } + + public void Reset() + { + this.stream.SetLength(0); + this.stream.Position = 0; + } + + public void WriteBuffer(Dictionary dict) + { + this.writer.WriteStartDocument(); + foreach (LayoutColumn c in this.layout.Columns) + { + this.LayoutCodeSwitch(c.Path, c.TypeArg, dict[c.Path]); + } + + this.writer.WriteEndDocument(); + } + + public void Dispose() + { + this.writer.Dispose(); + this.stream.Dispose(); + } + + private void LayoutCodeSwitch(UtfAnyString path, TypeArgument typeArg, object value) + { + if (!path.IsNull) + { + this.writer.WriteName(path); + } + + switch (typeArg.Type.LayoutCode) + { + case LayoutCode.Null: + this.writer.WriteNull(); + return; + + case LayoutCode.Boolean: + this.writer.WriteBoolean(value == null ? default(bool) : (bool)value); + return; + + case LayoutCode.Int8: + this.writer.WriteInt32(value == null ? default(sbyte) : (sbyte)value); + return; + + case LayoutCode.Int16: + this.writer.WriteInt32(value == null ? default(short) : (short)value); + return; + + case LayoutCode.Int32: + this.writer.WriteInt32(value == null ? default(int) : (int)value); + return; + + case LayoutCode.Int64: + this.writer.WriteInt64(value == null ? default(long) : (long)value); + return; + + case LayoutCode.UInt8: + this.writer.WriteInt32(value == null ? default(byte) : (byte)value); + return; + + case LayoutCode.UInt16: + this.writer.WriteInt32(value == null ? default(ushort) : (ushort)value); + return; + + case LayoutCode.UInt32: + this.writer.WriteInt32(value == null ? default(int) : unchecked((int)(uint)value)); + return; + + case LayoutCode.UInt64: + this.writer.WriteInt64(value == null ? default(long) : unchecked((long)(ulong)value)); + return; + + case LayoutCode.VarInt: + this.writer.WriteInt64(value == null ? default(long) : (long)value); + return; + + case LayoutCode.VarUInt: + this.writer.WriteInt64(value == null ? default(long) : unchecked((long)(ulong)value)); + return; + + case LayoutCode.Float32: + this.writer.WriteDouble(value == null ? default(float) : (float)value); + return; + + case LayoutCode.Float64: + this.writer.WriteDouble(value == null ? default(double) : (double)value); + return; + + case LayoutCode.Float128: + Decimal128 d128 = default(Decimal128); + if (value != null) + { + Float128 f128 = (Float128)value; + d128 = unchecked(Decimal128.FromIEEEBits((ulong)f128.High, (ulong)f128.Low)); + } + + this.writer.WriteDecimal128(d128); + return; + + case LayoutCode.Decimal: + this.writer.WriteDecimal128(value == null ? default(Decimal128) : new Decimal128((decimal)value)); + return; + + case LayoutCode.DateTime: + this.writer.WriteDateTime(value == null ? default(long) : ((DateTime)value).Ticks); + return; + + case LayoutCode.UnixDateTime: + this.writer.WriteDateTime(value == null ? default(long) : ((UnixDateTime)value).Milliseconds); + return; + + case LayoutCode.Guid: + this.writer.WriteString(value == null ? string.Empty : ((Guid)value).ToString()); + return; + + case LayoutCode.MongoDbObjectId: + this.writer.WriteObjectId(value == null ? default(ObjectId) : new ObjectId(((MongoDbObjectId)value).ToByteArray())); + return; + + case LayoutCode.Utf8: + this.writer.WriteString(value == null ? string.Empty : ((Utf8String)value).ToString()); + return; + + case LayoutCode.Binary: + this.writer.WriteBytes(value == null ? default(byte[]) : (byte[])value); + return; + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + this.DispatchObject(typeArg, value); + return; + + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + this.DispatchArray(typeArg, value); + return; + + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + this.DispatchSet(typeArg, value); + return; + + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + this.DispatchMap(typeArg, value); + return; + + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + this.DispatchTuple(typeArg, value); + return; + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + this.DispatchNullable(typeArg, value); + return; + + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + this.DispatchUDT(typeArg, value); + return; + + default: + Contract.Assert(false, $"Unknown type will be ignored: {typeArg}"); + return; + } + } + + private void DispatchObject(TypeArgument typeArg, object value) + { + this.writer.WriteStartDocument(); + + // TODO: support properties in an object scope. + this.writer.WriteEndDocument(); + } + + private void DispatchArray(TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 1); + + this.writer.WriteStartArray(); + foreach (object item in (List)value) + { + this.LayoutCodeSwitch(null, typeArg.TypeArgs[0], item); + } + + this.writer.WriteEndArray(); + } + + private void DispatchTuple(TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count >= 2); + List items = (List)value; + Contract.Assert(items.Count == typeArg.TypeArgs.Count); + + this.writer.WriteStartArray(); + for (int i = 0; i < items.Count; i++) + { + object item = items[i]; + this.LayoutCodeSwitch(null, typeArg.TypeArgs[i], item); + } + + this.writer.WriteEndArray(); + } + + private void DispatchNullable(TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 1); + + if (value != null) + { + this.LayoutCodeSwitch(null, typeArg.TypeArgs[0], value); + } + else + { + this.writer.WriteNull(); + } + } + + private void DispatchSet(TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 1); + + this.writer.WriteStartArray(); + foreach (object item in (List)value) + { + this.LayoutCodeSwitch(null, typeArg.TypeArgs[0], item); + } + + this.writer.WriteEndArray(); + } + + private void DispatchMap(TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 2); + + this.writer.WriteStartArray(); + foreach (object item in (List)value) + { + this.DispatchTuple(typeArg, item); + } + + this.writer.WriteEndArray(); + } + + private void DispatchUDT(TypeArgument typeArg, object value) + { + this.writer.WriteStartDocument(); + + Dictionary dict = (Dictionary)value; + Layout udt = this.resolver.Resolve(typeArg.TypeArgs.SchemaId); + foreach (LayoutColumn c in udt.Columns) + { + this.LayoutCodeSwitch(c.Path, c.TypeArg, dict[c.Path]); + } + + this.writer.WriteEndDocument(); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/CassandraProto/CassandraHotelSchema.cs b/dotnet/src/HybridRow.Tests.Perf/CassandraProto/CassandraHotelSchema.cs new file mode 100644 index 0000000..f8d73de --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/CassandraProto/CassandraHotelSchema.cs @@ -0,0 +1,1283 @@ +#pragma warning disable DontUseNamespaceAliases // Namespace Aliases should be avoided +#pragma warning disable NamespaceMatchesFolderStructure // Namespace Declarations must match folder structure +// +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: TestData/CassandraHotelSchema.proto +// +#pragma warning disable 1591, 0612, 3021 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf { + + /// Holder for reflection information generated from TestData/CassandraHotelSchema.proto + public static partial class CassandraHotelSchemaReflection { + + #region Descriptor + /// File descriptor for TestData/CassandraHotelSchema.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static CassandraHotelSchemaReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "CiNUZXN0RGF0YS9DYXNzYW5kcmFIb3RlbFNjaGVtYS5wcm90bxJITWljcm9z", + "b2Z0LkF6dXJlLkNvc21vcy5TZXJpYWxpemF0aW9uLkh5YnJpZFJvdy5UZXN0", + "cy5QZXJmLkNhc3NhbmRyYUhvdGVsGh5nb29nbGUvcHJvdG9idWYvd3JhcHBl", + "cnMucHJvdG8iYgoKUG9zdGFsQ29kZRIoCgN6aXAYASABKAsyGy5nb29nbGUu", + "cHJvdG9idWYuSW50MzJWYWx1ZRIqCgVwbHVzNBgCIAEoCzIbLmdvb2dsZS5w", + "cm90b2J1Zi5JbnQzMlZhbHVlIvsBCgdBZGRyZXNzEiwKBnN0cmVldBgBIAEo", + "CzIcLmdvb2dsZS5wcm90b2J1Zi5TdHJpbmdWYWx1ZRIqCgRjaXR5GAIgASgL", + "MhwuZ29vZ2xlLnByb3RvYnVmLlN0cmluZ1ZhbHVlEisKBXN0YXRlGAMgASgL", + "MhwuZ29vZ2xlLnByb3RvYnVmLlN0cmluZ1ZhbHVlEmkKC3Bvc3RhbF9jb2Rl", + "GAQgASgLMlQuTWljcm9zb2Z0LkF6dXJlLkNvc21vcy5TZXJpYWxpemF0aW9u", + "Lkh5YnJpZFJvdy5UZXN0cy5QZXJmLkNhc3NhbmRyYUhvdGVsLlBvc3RhbENv", + "ZGUi9QEKBkhvdGVscxIuCghob3RlbF9pZBgBIAEoCzIcLmdvb2dsZS5wcm90", + "b2J1Zi5TdHJpbmdWYWx1ZRIqCgRuYW1lGAIgASgLMhwuZ29vZ2xlLnByb3Rv", + "YnVmLlN0cmluZ1ZhbHVlEisKBXBob25lGAMgASgLMhwuZ29vZ2xlLnByb3Rv", + "YnVmLlN0cmluZ1ZhbHVlEmIKB2FkZHJlc3MYBCABKAsyUS5NaWNyb3NvZnQu", + "QXp1cmUuQ29zbW9zLlNlcmlhbGl6YXRpb24uSHlicmlkUm93LlRlc3RzLlBl", + "cmYuQ2Fzc2FuZHJhSG90ZWwuQWRkcmVzcyLeAQodQXZhaWxhYmxlX1Jvb21z", + "X0J5X0hvdGVsX0RhdGUSLgoIaG90ZWxfaWQYASABKAsyHC5nb29nbGUucHJv", + "dG9idWYuU3RyaW5nVmFsdWUSKQoEZGF0ZRgCIAEoCzIbLmdvb2dsZS5wcm90", + "b2J1Zi5JbnQ2NFZhbHVlEjAKC3Jvb21fbnVtYmVyGAMgASgLMhsuZ29vZ2xl", + "LnByb3RvYnVmLkludDMyVmFsdWUSMAoMaXNfYXZhaWxhYmxlGAQgASgLMhou", + "Z29vZ2xlLnByb3RvYnVmLkJvb2xWYWx1ZSKfBAoGR3Vlc3RzEi4KCGd1ZXN0", + "X2lkGAEgASgLMhwuZ29vZ2xlLnByb3RvYnVmLlN0cmluZ1ZhbHVlEjAKCmZp", + "cnN0X25hbWUYAiABKAsyHC5nb29nbGUucHJvdG9idWYuU3RyaW5nVmFsdWUS", + "LwoJbGFzdF9uYW1lGAMgASgLMhwuZ29vZ2xlLnByb3RvYnVmLlN0cmluZ1Zh", + "bHVlEisKBXRpdGxlGAQgASgLMhwuZ29vZ2xlLnByb3RvYnVmLlN0cmluZ1Zh", + "bHVlEg4KBmVtYWlscxgFIAMoCRIVCg1waG9uZV9udW1iZXJzGAYgAygJEnIK", + "CWFkZHJlc3NlcxgHIAMoCzJfLk1pY3Jvc29mdC5BenVyZS5Db3Ntb3MuU2Vy", + "aWFsaXphdGlvbi5IeWJyaWRSb3cuVGVzdHMuUGVyZi5DYXNzYW5kcmFIb3Rl", + "bC5HdWVzdHMuQWRkcmVzc2VzRW50cnkSNAoOY29uZmlybV9udW1iZXIYCCAB", + "KAsyHC5nb29nbGUucHJvdG9idWYuU3RyaW5nVmFsdWUagwEKDkFkZHJlc3Nl", + "c0VudHJ5EgsKA2tleRgBIAEoCRJgCgV2YWx1ZRgCIAEoCzJRLk1pY3Jvc29m", + "dC5BenVyZS5Db3Ntb3MuU2VyaWFsaXphdGlvbi5IeWJyaWRSb3cuVGVzdHMu", + "UGVyZi5DYXNzYW5kcmFIb3RlbC5BZGRyZXNzOgI4AUJUqgJRTWljcm9zb2Z0", + "LkF6dXJlLkNvc21vcy5TZXJpYWxpemF0aW9uLkh5YnJpZFJvdy5UZXN0cy5Q", + "ZXJmLkNhc3NhbmRyYUhvdGVsLlByb3RvYnVmYgZwcm90bzM=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, }, + new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.PostalCode), global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.PostalCode.Parser, new[]{ "Zip", "Plus4" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address), global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address.Parser, new[]{ "Street", "City", "State", "PostalCode" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Hotels), global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Hotels.Parser, new[]{ "HotelId", "Name", "Phone", "Address" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Available_Rooms_By_Hotel_Date), global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Available_Rooms_By_Hotel_Date.Parser, new[]{ "HotelId", "Date", "RoomNumber", "IsAvailable" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Guests), global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Guests.Parser, new[]{ "GuestId", "FirstName", "LastName", "Title", "Emails", "PhoneNumbers", "Addresses", "ConfirmNumber" }, null, null, new pbr::GeneratedClrTypeInfo[] { null, }) + })); + } + #endregion + + } + #region Messages + public sealed partial class PostalCode : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new PostalCode()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.CassandraHotelSchemaReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public PostalCode() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public PostalCode(PostalCode other) : this() { + Zip = other.Zip; + Plus4 = other.Plus4; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public PostalCode Clone() { + return new PostalCode(this); + } + + /// Field number for the "zip" field. + public const int ZipFieldNumber = 1; + private static readonly pb::FieldCodec _single_zip_codec = pb::FieldCodec.ForStructWrapper(10); + private int? zip_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int? Zip { + get { return zip_; } + set { + zip_ = value; + } + } + + /// Field number for the "plus4" field. + public const int Plus4FieldNumber = 2; + private static readonly pb::FieldCodec _single_plus4_codec = pb::FieldCodec.ForStructWrapper(18); + private int? plus4_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int? Plus4 { + get { return plus4_; } + set { + plus4_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as PostalCode); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(PostalCode other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Zip != other.Zip) return false; + if (Plus4 != other.Plus4) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (zip_ != null) hash ^= Zip.GetHashCode(); + if (plus4_ != null) hash ^= Plus4.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (zip_ != null) { + _single_zip_codec.WriteTagAndValue(output, Zip); + } + if (plus4_ != null) { + _single_plus4_codec.WriteTagAndValue(output, Plus4); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (zip_ != null) { + size += _single_zip_codec.CalculateSizeWithTag(Zip); + } + if (plus4_ != null) { + size += _single_plus4_codec.CalculateSizeWithTag(Plus4); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(PostalCode other) { + if (other == null) { + return; + } + if (other.zip_ != null) { + if (zip_ == null || other.Zip != 0) { + Zip = other.Zip; + } + } + if (other.plus4_ != null) { + if (plus4_ == null || other.Plus4 != 0) { + Plus4 = other.Plus4; + } + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + int? value = _single_zip_codec.Read(input); + if (zip_ == null || value != 0) { + Zip = value; + } + break; + } + case 18: { + int? value = _single_plus4_codec.Read(input); + if (plus4_ == null || value != 0) { + Plus4 = value; + } + break; + } + } + } + } + + } + + public sealed partial class Address : pb::IMessage
{ + private static readonly pb::MessageParser
_parser = new pb::MessageParser
(() => new Address()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser
Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.CassandraHotelSchemaReflection.Descriptor.MessageTypes[1]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Address() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Address(Address other) : this() { + Street = other.Street; + City = other.City; + State = other.State; + PostalCode = other.postalCode_ != null ? other.PostalCode.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Address Clone() { + return new Address(this); + } + + /// Field number for the "street" field. + public const int StreetFieldNumber = 1; + private static readonly pb::FieldCodec _single_street_codec = pb::FieldCodec.ForClassWrapper(10); + private string street_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string Street { + get { return street_; } + set { + street_ = value; + } + } + + /// Field number for the "city" field. + public const int CityFieldNumber = 2; + private static readonly pb::FieldCodec _single_city_codec = pb::FieldCodec.ForClassWrapper(18); + private string city_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string City { + get { return city_; } + set { + city_ = value; + } + } + + /// Field number for the "state" field. + public const int StateFieldNumber = 3; + private static readonly pb::FieldCodec _single_state_codec = pb::FieldCodec.ForClassWrapper(26); + private string state_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string State { + get { return state_; } + set { + state_ = value; + } + } + + /// Field number for the "postal_code" field. + public const int PostalCodeFieldNumber = 4; + private global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.PostalCode postalCode_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.PostalCode PostalCode { + get { return postalCode_; } + set { + postalCode_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as Address); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(Address other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Street != other.Street) return false; + if (City != other.City) return false; + if (State != other.State) return false; + if (!object.Equals(PostalCode, other.PostalCode)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (street_ != null) hash ^= Street.GetHashCode(); + if (city_ != null) hash ^= City.GetHashCode(); + if (state_ != null) hash ^= State.GetHashCode(); + if (postalCode_ != null) hash ^= PostalCode.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (street_ != null) { + _single_street_codec.WriteTagAndValue(output, Street); + } + if (city_ != null) { + _single_city_codec.WriteTagAndValue(output, City); + } + if (state_ != null) { + _single_state_codec.WriteTagAndValue(output, State); + } + if (postalCode_ != null) { + output.WriteRawTag(34); + output.WriteMessage(PostalCode); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (street_ != null) { + size += _single_street_codec.CalculateSizeWithTag(Street); + } + if (city_ != null) { + size += _single_city_codec.CalculateSizeWithTag(City); + } + if (state_ != null) { + size += _single_state_codec.CalculateSizeWithTag(State); + } + if (postalCode_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(PostalCode); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(Address other) { + if (other == null) { + return; + } + if (other.street_ != null) { + if (street_ == null || other.Street != "") { + Street = other.Street; + } + } + if (other.city_ != null) { + if (city_ == null || other.City != "") { + City = other.City; + } + } + if (other.state_ != null) { + if (state_ == null || other.State != "") { + State = other.State; + } + } + if (other.postalCode_ != null) { + if (postalCode_ == null) { + postalCode_ = new global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.PostalCode(); + } + PostalCode.MergeFrom(other.PostalCode); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + string value = _single_street_codec.Read(input); + if (street_ == null || value != "") { + Street = value; + } + break; + } + case 18: { + string value = _single_city_codec.Read(input); + if (city_ == null || value != "") { + City = value; + } + break; + } + case 26: { + string value = _single_state_codec.Read(input); + if (state_ == null || value != "") { + State = value; + } + break; + } + case 34: { + if (postalCode_ == null) { + postalCode_ = new global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.PostalCode(); + } + input.ReadMessage(postalCode_); + break; + } + } + } + } + + } + + public sealed partial class Hotels : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Hotels()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.CassandraHotelSchemaReflection.Descriptor.MessageTypes[2]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Hotels() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Hotels(Hotels other) : this() { + HotelId = other.HotelId; + Name = other.Name; + Phone = other.Phone; + Address = other.address_ != null ? other.Address.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Hotels Clone() { + return new Hotels(this); + } + + /// Field number for the "hotel_id" field. + public const int HotelIdFieldNumber = 1; + private static readonly pb::FieldCodec _single_hotelId_codec = pb::FieldCodec.ForClassWrapper(10); + private string hotelId_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string HotelId { + get { return hotelId_; } + set { + hotelId_ = value; + } + } + + /// Field number for the "name" field. + public const int NameFieldNumber = 2; + private static readonly pb::FieldCodec _single_name_codec = pb::FieldCodec.ForClassWrapper(18); + private string name_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string Name { + get { return name_; } + set { + name_ = value; + } + } + + /// Field number for the "phone" field. + public const int PhoneFieldNumber = 3; + private static readonly pb::FieldCodec _single_phone_codec = pb::FieldCodec.ForClassWrapper(26); + private string phone_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string Phone { + get { return phone_; } + set { + phone_ = value; + } + } + + /// Field number for the "address" field. + public const int AddressFieldNumber = 4; + private global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address address_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address Address { + get { return address_; } + set { + address_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as Hotels); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(Hotels other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (HotelId != other.HotelId) return false; + if (Name != other.Name) return false; + if (Phone != other.Phone) return false; + if (!object.Equals(Address, other.Address)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (hotelId_ != null) hash ^= HotelId.GetHashCode(); + if (name_ != null) hash ^= Name.GetHashCode(); + if (phone_ != null) hash ^= Phone.GetHashCode(); + if (address_ != null) hash ^= Address.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (hotelId_ != null) { + _single_hotelId_codec.WriteTagAndValue(output, HotelId); + } + if (name_ != null) { + _single_name_codec.WriteTagAndValue(output, Name); + } + if (phone_ != null) { + _single_phone_codec.WriteTagAndValue(output, Phone); + } + if (address_ != null) { + output.WriteRawTag(34); + output.WriteMessage(Address); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (hotelId_ != null) { + size += _single_hotelId_codec.CalculateSizeWithTag(HotelId); + } + if (name_ != null) { + size += _single_name_codec.CalculateSizeWithTag(Name); + } + if (phone_ != null) { + size += _single_phone_codec.CalculateSizeWithTag(Phone); + } + if (address_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(Address); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(Hotels other) { + if (other == null) { + return; + } + if (other.hotelId_ != null) { + if (hotelId_ == null || other.HotelId != "") { + HotelId = other.HotelId; + } + } + if (other.name_ != null) { + if (name_ == null || other.Name != "") { + Name = other.Name; + } + } + if (other.phone_ != null) { + if (phone_ == null || other.Phone != "") { + Phone = other.Phone; + } + } + if (other.address_ != null) { + if (address_ == null) { + address_ = new global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address(); + } + Address.MergeFrom(other.Address); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + string value = _single_hotelId_codec.Read(input); + if (hotelId_ == null || value != "") { + HotelId = value; + } + break; + } + case 18: { + string value = _single_name_codec.Read(input); + if (name_ == null || value != "") { + Name = value; + } + break; + } + case 26: { + string value = _single_phone_codec.Read(input); + if (phone_ == null || value != "") { + Phone = value; + } + break; + } + case 34: { + if (address_ == null) { + address_ = new global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address(); + } + input.ReadMessage(address_); + break; + } + } + } + } + + } + + public sealed partial class Available_Rooms_By_Hotel_Date : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Available_Rooms_By_Hotel_Date()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.CassandraHotelSchemaReflection.Descriptor.MessageTypes[3]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Available_Rooms_By_Hotel_Date() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Available_Rooms_By_Hotel_Date(Available_Rooms_By_Hotel_Date other) : this() { + HotelId = other.HotelId; + Date = other.Date; + RoomNumber = other.RoomNumber; + IsAvailable = other.IsAvailable; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Available_Rooms_By_Hotel_Date Clone() { + return new Available_Rooms_By_Hotel_Date(this); + } + + /// Field number for the "hotel_id" field. + public const int HotelIdFieldNumber = 1; + private static readonly pb::FieldCodec _single_hotelId_codec = pb::FieldCodec.ForClassWrapper(10); + private string hotelId_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string HotelId { + get { return hotelId_; } + set { + hotelId_ = value; + } + } + + /// Field number for the "date" field. + public const int DateFieldNumber = 2; + private static readonly pb::FieldCodec _single_date_codec = pb::FieldCodec.ForStructWrapper(18); + private long? date_; + /// + /// datetime + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public long? Date { + get { return date_; } + set { + date_ = value; + } + } + + /// Field number for the "room_number" field. + public const int RoomNumberFieldNumber = 3; + private static readonly pb::FieldCodec _single_roomNumber_codec = pb::FieldCodec.ForStructWrapper(26); + private int? roomNumber_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int? RoomNumber { + get { return roomNumber_; } + set { + roomNumber_ = value; + } + } + + /// Field number for the "is_available" field. + public const int IsAvailableFieldNumber = 4; + private static readonly pb::FieldCodec _single_isAvailable_codec = pb::FieldCodec.ForStructWrapper(34); + private bool? isAvailable_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool? IsAvailable { + get { return isAvailable_; } + set { + isAvailable_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as Available_Rooms_By_Hotel_Date); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(Available_Rooms_By_Hotel_Date other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (HotelId != other.HotelId) return false; + if (Date != other.Date) return false; + if (RoomNumber != other.RoomNumber) return false; + if (IsAvailable != other.IsAvailable) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (hotelId_ != null) hash ^= HotelId.GetHashCode(); + if (date_ != null) hash ^= Date.GetHashCode(); + if (roomNumber_ != null) hash ^= RoomNumber.GetHashCode(); + if (isAvailable_ != null) hash ^= IsAvailable.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (hotelId_ != null) { + _single_hotelId_codec.WriteTagAndValue(output, HotelId); + } + if (date_ != null) { + _single_date_codec.WriteTagAndValue(output, Date); + } + if (roomNumber_ != null) { + _single_roomNumber_codec.WriteTagAndValue(output, RoomNumber); + } + if (isAvailable_ != null) { + _single_isAvailable_codec.WriteTagAndValue(output, IsAvailable); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (hotelId_ != null) { + size += _single_hotelId_codec.CalculateSizeWithTag(HotelId); + } + if (date_ != null) { + size += _single_date_codec.CalculateSizeWithTag(Date); + } + if (roomNumber_ != null) { + size += _single_roomNumber_codec.CalculateSizeWithTag(RoomNumber); + } + if (isAvailable_ != null) { + size += _single_isAvailable_codec.CalculateSizeWithTag(IsAvailable); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(Available_Rooms_By_Hotel_Date other) { + if (other == null) { + return; + } + if (other.hotelId_ != null) { + if (hotelId_ == null || other.HotelId != "") { + HotelId = other.HotelId; + } + } + if (other.date_ != null) { + if (date_ == null || other.Date != 0L) { + Date = other.Date; + } + } + if (other.roomNumber_ != null) { + if (roomNumber_ == null || other.RoomNumber != 0) { + RoomNumber = other.RoomNumber; + } + } + if (other.isAvailable_ != null) { + if (isAvailable_ == null || other.IsAvailable != false) { + IsAvailable = other.IsAvailable; + } + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + string value = _single_hotelId_codec.Read(input); + if (hotelId_ == null || value != "") { + HotelId = value; + } + break; + } + case 18: { + long? value = _single_date_codec.Read(input); + if (date_ == null || value != 0L) { + Date = value; + } + break; + } + case 26: { + int? value = _single_roomNumber_codec.Read(input); + if (roomNumber_ == null || value != 0) { + RoomNumber = value; + } + break; + } + case 34: { + bool? value = _single_isAvailable_codec.Read(input); + if (isAvailable_ == null || value != false) { + IsAvailable = value; + } + break; + } + } + } + } + + } + + public sealed partial class Guests : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Guests()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.CassandraHotelSchemaReflection.Descriptor.MessageTypes[4]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Guests() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Guests(Guests other) : this() { + GuestId = other.GuestId; + FirstName = other.FirstName; + LastName = other.LastName; + Title = other.Title; + emails_ = other.emails_.Clone(); + phoneNumbers_ = other.phoneNumbers_.Clone(); + addresses_ = other.addresses_.Clone(); + ConfirmNumber = other.ConfirmNumber; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Guests Clone() { + return new Guests(this); + } + + /// Field number for the "guest_id" field. + public const int GuestIdFieldNumber = 1; + private static readonly pb::FieldCodec _single_guestId_codec = pb::FieldCodec.ForClassWrapper(10); + private string guestId_; + /// + /// guid + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string GuestId { + get { return guestId_; } + set { + guestId_ = value; + } + } + + /// Field number for the "first_name" field. + public const int FirstNameFieldNumber = 2; + private static readonly pb::FieldCodec _single_firstName_codec = pb::FieldCodec.ForClassWrapper(18); + private string firstName_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string FirstName { + get { return firstName_; } + set { + firstName_ = value; + } + } + + /// Field number for the "last_name" field. + public const int LastNameFieldNumber = 3; + private static readonly pb::FieldCodec _single_lastName_codec = pb::FieldCodec.ForClassWrapper(26); + private string lastName_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string LastName { + get { return lastName_; } + set { + lastName_ = value; + } + } + + /// Field number for the "title" field. + public const int TitleFieldNumber = 4; + private static readonly pb::FieldCodec _single_title_codec = pb::FieldCodec.ForClassWrapper(34); + private string title_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string Title { + get { return title_; } + set { + title_ = value; + } + } + + /// Field number for the "emails" field. + public const int EmailsFieldNumber = 5; + private static readonly pb::FieldCodec _repeated_emails_codec + = pb::FieldCodec.ForString(42); + private readonly pbc::RepeatedField emails_ = new pbc::RepeatedField(); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public pbc::RepeatedField Emails { + get { return emails_; } + } + + /// Field number for the "phone_numbers" field. + public const int PhoneNumbersFieldNumber = 6; + private static readonly pb::FieldCodec _repeated_phoneNumbers_codec + = pb::FieldCodec.ForString(50); + private readonly pbc::RepeatedField phoneNumbers_ = new pbc::RepeatedField(); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public pbc::RepeatedField PhoneNumbers { + get { return phoneNumbers_; } + } + + /// Field number for the "addresses" field. + public const int AddressesFieldNumber = 7; + private static readonly pbc::MapField.Codec _map_addresses_codec + = new pbc::MapField.Codec(pb::FieldCodec.ForString(10), pb::FieldCodec.ForMessage(18, global::Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf.Address.Parser), 58); + private readonly pbc::MapField addresses_ = new pbc::MapField(); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public pbc::MapField Addresses { + get { return addresses_; } + } + + /// Field number for the "confirm_number" field. + public const int ConfirmNumberFieldNumber = 8; + private static readonly pb::FieldCodec _single_confirmNumber_codec = pb::FieldCodec.ForClassWrapper(66); + private string confirmNumber_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string ConfirmNumber { + get { return confirmNumber_; } + set { + confirmNumber_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as Guests); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(Guests other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (GuestId != other.GuestId) return false; + if (FirstName != other.FirstName) return false; + if (LastName != other.LastName) return false; + if (Title != other.Title) return false; + if(!emails_.Equals(other.emails_)) return false; + if(!phoneNumbers_.Equals(other.phoneNumbers_)) return false; + if (!Addresses.Equals(other.Addresses)) return false; + if (ConfirmNumber != other.ConfirmNumber) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (guestId_ != null) hash ^= GuestId.GetHashCode(); + if (firstName_ != null) hash ^= FirstName.GetHashCode(); + if (lastName_ != null) hash ^= LastName.GetHashCode(); + if (title_ != null) hash ^= Title.GetHashCode(); + hash ^= emails_.GetHashCode(); + hash ^= phoneNumbers_.GetHashCode(); + hash ^= Addresses.GetHashCode(); + if (confirmNumber_ != null) hash ^= ConfirmNumber.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (guestId_ != null) { + _single_guestId_codec.WriteTagAndValue(output, GuestId); + } + if (firstName_ != null) { + _single_firstName_codec.WriteTagAndValue(output, FirstName); + } + if (lastName_ != null) { + _single_lastName_codec.WriteTagAndValue(output, LastName); + } + if (title_ != null) { + _single_title_codec.WriteTagAndValue(output, Title); + } + emails_.WriteTo(output, _repeated_emails_codec); + phoneNumbers_.WriteTo(output, _repeated_phoneNumbers_codec); + addresses_.WriteTo(output, _map_addresses_codec); + if (confirmNumber_ != null) { + _single_confirmNumber_codec.WriteTagAndValue(output, ConfirmNumber); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (guestId_ != null) { + size += _single_guestId_codec.CalculateSizeWithTag(GuestId); + } + if (firstName_ != null) { + size += _single_firstName_codec.CalculateSizeWithTag(FirstName); + } + if (lastName_ != null) { + size += _single_lastName_codec.CalculateSizeWithTag(LastName); + } + if (title_ != null) { + size += _single_title_codec.CalculateSizeWithTag(Title); + } + size += emails_.CalculateSize(_repeated_emails_codec); + size += phoneNumbers_.CalculateSize(_repeated_phoneNumbers_codec); + size += addresses_.CalculateSize(_map_addresses_codec); + if (confirmNumber_ != null) { + size += _single_confirmNumber_codec.CalculateSizeWithTag(ConfirmNumber); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(Guests other) { + if (other == null) { + return; + } + if (other.guestId_ != null) { + if (guestId_ == null || other.GuestId != "") { + GuestId = other.GuestId; + } + } + if (other.firstName_ != null) { + if (firstName_ == null || other.FirstName != "") { + FirstName = other.FirstName; + } + } + if (other.lastName_ != null) { + if (lastName_ == null || other.LastName != "") { + LastName = other.LastName; + } + } + if (other.title_ != null) { + if (title_ == null || other.Title != "") { + Title = other.Title; + } + } + emails_.Add(other.emails_); + phoneNumbers_.Add(other.phoneNumbers_); + addresses_.Add(other.addresses_); + if (other.confirmNumber_ != null) { + if (confirmNumber_ == null || other.ConfirmNumber != "") { + ConfirmNumber = other.ConfirmNumber; + } + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + string value = _single_guestId_codec.Read(input); + if (guestId_ == null || value != "") { + GuestId = value; + } + break; + } + case 18: { + string value = _single_firstName_codec.Read(input); + if (firstName_ == null || value != "") { + FirstName = value; + } + break; + } + case 26: { + string value = _single_lastName_codec.Read(input); + if (lastName_ == null || value != "") { + LastName = value; + } + break; + } + case 34: { + string value = _single_title_codec.Read(input); + if (title_ == null || value != "") { + Title = value; + } + break; + } + case 42: { + emails_.AddEntriesFrom(input, _repeated_emails_codec); + break; + } + case 50: { + phoneNumbers_.AddEntriesFrom(input, _repeated_phoneNumbers_codec); + break; + } + case 58: { + addresses_.AddEntriesFrom(input, _map_addresses_codec); + break; + } + case 66: { + string value = _single_confirmNumber_codec.Read(input); + if (confirmNumber_ == null || value != "") { + ConfirmNumber = value; + } + break; + } + } + } + } + + } + + #endregion + +} + +#endregion Designer generated code diff --git a/dotnet/src/HybridRow.Tests.Perf/CodeGenMicroBenchmarkSuite.cs b/dotnet/src/HybridRow.Tests.Perf/CodeGenMicroBenchmarkSuite.cs new file mode 100644 index 0000000..670045a --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/CodeGenMicroBenchmarkSuite.cs @@ -0,0 +1,293 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System.Collections.Generic; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + /// + /// Tests involving generated (early bound) code compiled from schema based on a partial implementation + /// of Cassandra Hotel Schema described here: https://www.oreilly.com/ideas/cassandra-data-modeling . + /// + /// The tests here differ from in that they rely on + /// the schema being known at compile time instead of runtime. This allows code to be generated that + /// directly addresses the schema structure instead of dynamically discovering schema structure at + /// runtime. + /// + /// + [TestClass] + public sealed class CodeGenMicroBenchmarkSuite : MicroBenchmarkSuiteBase + { + private const int GuestCount = 1000; + private const int HotelCount = 10000; + private const int RoomsCount = 10000; + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task ProtobufGuestsWriteBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.ProtobufWriteBenchmark("Guests", "Guests", CodeGenMicroBenchmarkSuite.GuestCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task ProtobufHotelWriteBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.ProtobufWriteBenchmark("Hotels", "Hotels", CodeGenMicroBenchmarkSuite.HotelCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task ProtobufRoomsWriteBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.ProtobufWriteBenchmark( + "Available_Rooms_By_Hotel_Date", + "Rooms", + CodeGenMicroBenchmarkSuite.RoomsCount, + expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task ProtobufGuestsReadBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.ProtobufReadBenchmark("Guests", "Guests", CodeGenMicroBenchmarkSuite.GuestCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task ProtobufHotelReadBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.ProtobufReadBenchmark("Hotels", "Hotels", CodeGenMicroBenchmarkSuite.HotelCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task ProtobufRoomsReadBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.ProtobufReadBenchmark( + "Available_Rooms_By_Hotel_Date", + "Rooms", + CodeGenMicroBenchmarkSuite.RoomsCount, + expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task CodeGenGuestsWriteBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.CodeGenWriteBenchmark(resolver, "Guests", "Guests", CodeGenMicroBenchmarkSuite.GuestCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task CodeGenHotelWriteBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.CodeGenWriteBenchmark(resolver, "Hotels", "Hotels", CodeGenMicroBenchmarkSuite.HotelCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task CodeGenRoomsWriteBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.CodeGenWriteBenchmark( + resolver, + "Available_Rooms_By_Hotel_Date", + "Rooms", + CodeGenMicroBenchmarkSuite.RoomsCount, + expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task CodeGenGuestsReadBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.CodeGenReadBenchmark(resolver, "Guests", "Guests", CodeGenMicroBenchmarkSuite.GuestCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task CodeGenHotelReadBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.CodeGenReadBenchmark(resolver, "Hotels", "Hotels", CodeGenMicroBenchmarkSuite.HotelCount, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task CodeGenRoomsReadBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + CodeGenMicroBenchmarkSuite.CodeGenReadBenchmark( + resolver, + "Available_Rooms_By_Hotel_Date", + "Rooms", + CodeGenMicroBenchmarkSuite.RoomsCount, + expected); + } + + private static void ProtobufWriteBenchmark( + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + BenchmarkContext context = new BenchmarkContext + { + ProtobufWriter = new ProtobufRowGenerator(schemaName, BenchmarkSuiteBase.InitialCapacity) + }; + + MicroBenchmarkSuiteBase.Benchmark( + "CodeGen", + "Write", + dataSetName, + "Protobuf", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, Dictionary tableValue) => { ctx.ProtobufWriter.WriteBuffer(tableValue); }, + (ref BenchmarkContext ctx, Dictionary tableValue) => ctx.ProtobufWriter.Length, + expected); + } + + private static void ProtobufReadBenchmark( + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + BenchmarkContext context = new BenchmarkContext + { + ProtobufWriter = new ProtobufRowGenerator(schemaName, BenchmarkSuiteBase.InitialCapacity) + }; + + foreach (Dictionary tableValue in expected) + { + context.ProtobufWriter.WriteBuffer(tableValue); + expectedSerialized.Add(context.ProtobufWriter.ToArray()); + } + + MicroBenchmarkSuiteBase.Benchmark( + "CodeGen", + "Read", + dataSetName, + "Protobuf", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, byte[] tableValue) => ctx.ProtobufWriter.ReadBuffer(tableValue), + (ref BenchmarkContext ctx, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + + private static void CodeGenWriteBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + CodeGenWriter = new CodeGenRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + MicroBenchmarkSuiteBase.Benchmark( + "CodeGen", + "Write", + dataSetName, + "HybridRowGen", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, Dictionary tableValue) => + { + ctx.CodeGenWriter.Reset(); + + Result r = ctx.CodeGenWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + }, + (ref BenchmarkContext ctx, Dictionary tableValue) => ctx.CodeGenWriter.Length, + expected); + } + + private static void CodeGenReadBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + CodeGenWriter = new CodeGenRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + foreach (Dictionary tableValue in expected) + { + context.CodeGenWriter.Reset(); + + Result r = context.CodeGenWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + expectedSerialized.Add(context.CodeGenWriter.ToArray()); + } + + MicroBenchmarkSuiteBase.Benchmark( + "CodeGen", + "Read", + dataSetName, + "HybridRowGen", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, byte[] tableValue) => + { + Result r = ctx.CodeGenWriter.ReadBuffer(tableValue); + ResultAssert.IsSuccess(r); + }, + (ref BenchmarkContext ctx, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/CodeGenRowGenerator.cs b/dotnet/src/HybridRow.Tests.Perf/CodeGenRowGenerator.cs new file mode 100644 index 0000000..40923ad --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/CodeGenRowGenerator.cs @@ -0,0 +1,872 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + internal ref struct CodeGenRowGenerator + { + private RowBuffer row; + private readonly HybridRowSerializer dispatcher; + + public CodeGenRowGenerator(int capacity, Layout layout, LayoutResolver resolver, ISpanResizer resizer = default) + { + this.row = new RowBuffer(capacity, resizer); + this.row.InitLayout(HybridRowVersion.V1, layout, resolver); + + switch (layout.Name) + { + case "Hotels": + this.dispatcher = new HotelsHybridRowSerializer(layout, resolver); + break; + + case "Guests": + this.dispatcher = new GuestsHybridRowSerializer(layout, resolver); + break; + + case "Available_Rooms_By_Hotel_Date": + this.dispatcher = new RoomsHybridRowSerializer(layout, resolver); + break; + + default: + Contract.Fail($"Unknown schema will be ignored: {layout.Name}"); + this.dispatcher = null; + break; + } + } + + public int Length => this.row.Length; + + public byte[] ToArray() + { + return this.row.ToArray(); + } + + public void Reset() + { + Layout layout = this.row.Resolver.Resolve(this.row.Header.SchemaId); + this.row.InitLayout(HybridRowVersion.V1, layout, this.row.Resolver); + } + + public Result WriteBuffer(Dictionary tableValue) + { + RowCursor root = RowCursor.Create(ref this.row); + return this.dispatcher.WriteBuffer(ref this.row, ref root, tableValue); + } + + public Result ReadBuffer(byte[] buffer) + { + this.row = new RowBuffer(buffer.AsSpan(), HybridRowVersion.V1, this.row.Resolver); + RowCursor root = RowCursor.Create(ref this.row); + return this.dispatcher.ReadBuffer(ref this.row, ref root); + } + + private abstract class HybridRowSerializer + { + public abstract Result WriteBuffer(ref RowBuffer row, ref RowCursor root, Dictionary tableValue); + + public abstract Result ReadBuffer(ref RowBuffer row, ref RowCursor root); + } + + private sealed class GuestsHybridRowSerializer : HybridRowSerializer + { + private static readonly Utf8String GuestIdName = Utf8String.TranscodeUtf16("guest_id"); + private static readonly Utf8String FirstNameName = Utf8String.TranscodeUtf16("first_name"); + private static readonly Utf8String LastNameName = Utf8String.TranscodeUtf16("last_name"); + private static readonly Utf8String TitleName = Utf8String.TranscodeUtf16("title"); + private static readonly Utf8String EmailsName = Utf8String.TranscodeUtf16("emails"); + private static readonly Utf8String PhoneNumbersName = Utf8String.TranscodeUtf16("phone_numbers"); + private static readonly Utf8String AddressesName = Utf8String.TranscodeUtf16("addresses"); + private static readonly Utf8String ConfirmNumberName = Utf8String.TranscodeUtf16("confirm_number"); + private readonly LayoutColumn guestId; + private readonly LayoutColumn firstName; + private readonly LayoutColumn lastName; + private readonly LayoutColumn title; + private readonly LayoutColumn emails; + private readonly LayoutColumn phoneNumbers; + private readonly LayoutColumn addresses; + private readonly LayoutColumn confirmNumber; + private readonly StringToken emailsToken; + private readonly StringToken phoneNumbersToken; + private readonly StringToken addressesToken; + private readonly TypeArgumentList addressesFieldType; + private readonly AddressHybridRowSerializer addressSerializer; + private readonly LayoutScope.WriterFunc> addressSerializerWriter; + + public GuestsHybridRowSerializer(Layout layout, LayoutResolver resolver) + { + layout.TryFind(GuestsHybridRowSerializer.GuestIdName, out this.guestId); + layout.TryFind(GuestsHybridRowSerializer.FirstNameName, out this.firstName); + layout.TryFind(GuestsHybridRowSerializer.LastNameName, out this.lastName); + layout.TryFind(GuestsHybridRowSerializer.TitleName, out this.title); + layout.TryFind(GuestsHybridRowSerializer.EmailsName, out this.emails); + layout.TryFind(GuestsHybridRowSerializer.PhoneNumbersName, out this.phoneNumbers); + layout.TryFind(GuestsHybridRowSerializer.AddressesName, out this.addresses); + layout.TryFind(GuestsHybridRowSerializer.ConfirmNumberName, out this.confirmNumber); + layout.Tokenizer.TryFindToken(this.emails.Path, out this.emailsToken); + layout.Tokenizer.TryFindToken(this.phoneNumbers.Path, out this.phoneNumbersToken); + layout.Tokenizer.TryFindToken(this.addresses.Path, out this.addressesToken); + + this.addressesFieldType = new TypeArgumentList( + new[] + { + new TypeArgument(LayoutType.TypedTuple, this.addresses.TypeArgs) + }); + + this.addressSerializer = new AddressHybridRowSerializer(resolver.Resolve(this.addresses.TypeArgs[1].TypeArgs.SchemaId), resolver); + this.addressSerializerWriter = this.addressSerializer.WriteBuffer; + } + + public override Result WriteBuffer(ref RowBuffer row, ref RowCursor root, Dictionary tableValue) + { + foreach ((Utf8String key, object value) in tableValue) + { + Result r; + switch (0) + { + case 0 when key.Equals(GuestsHybridRowSerializer.GuestIdName): + if (value != null) + { + r = LayoutType.Guid.WriteFixed(ref row, ref root, this.guestId, (Guid)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.FirstNameName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.firstName, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.LastNameName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.lastName, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.TitleName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.title, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.EmailsName): + if (value != null) + { + root.Find(ref row, this.emailsToken); + r = LayoutType.TypedArray.WriteScope( + ref row, + ref root, + this.emails.TypeArgs, + (List)value, + (ref RowBuffer row2, ref RowCursor childScope, List context) => + { + foreach (object item in context) + { + Result r2 = LayoutType.Utf8.WriteSparse(ref row2, ref childScope, (Utf8String)item); + if (r2 != Result.Success) + { + return r2; + } + + childScope.MoveNext(ref row2); + } + + return Result.Success; + }); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.PhoneNumbersName): + if (value != null) + { + root.Find(ref row, this.phoneNumbersToken); + r = LayoutType.TypedArray.WriteScope(ref row, ref root, this.phoneNumbers.TypeArgs, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + foreach (object item in (List)value) + { + r = LayoutType.Utf8.WriteSparse(ref row, ref childScope, (Utf8String)item); + if (r != Result.Success) + { + return r; + } + + childScope.MoveNext(ref row); + } + + root.Skip(ref row, ref childScope); + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.AddressesName): + if (value != null) + { + root.Find(ref row, this.addressesToken); + r = LayoutType.TypedMap.WriteScope( + ref row, + ref root, + this.addresses.TypeArgs, + (this, (List)value), + (ref RowBuffer row2, ref RowCursor childScope, (GuestsHybridRowSerializer _this, List value) ctx) => + { + foreach (object item in ctx.value) + { + Result r2 = LayoutType.TypedTuple.WriteScope( + ref row2, + ref childScope, + ctx._this.addresses.TypeArgs, + (ctx._this, (List)item), + (ref RowBuffer row3, ref RowCursor tupleScope, (GuestsHybridRowSerializer _this, List value) ctx2) => + { + Result r3 = LayoutType.Utf8.WriteSparse(ref row3, ref tupleScope, (Utf8String)ctx2.value[0]); + if (r3 != Result.Success) + { + return r3; + } + + tupleScope.MoveNext(ref row3); + return LayoutType.ImmutableUDT.WriteScope( + ref row3, + ref tupleScope, + ctx2._this.addresses.TypeArgs[1].TypeArgs, + (Dictionary)ctx2.value[1], + ctx2._this.addressSerializerWriter); + }); + if (r2 != Result.Success) + { + return r2; + } + + childScope.MoveNext(ref row2); + } + + return Result.Success; + }); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(GuestsHybridRowSerializer.ConfirmNumberName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.confirmNumber, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + default: + Contract.Fail($"Unknown property name: {key}"); + break; + } + } + + return Result.Success; + } + + public override Result ReadBuffer(ref RowBuffer row, ref RowCursor root) + { + Result r = LayoutType.Guid.ReadFixed(ref row, ref root, this.guestId, out Guid _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.firstName, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.lastName, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.title, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + root.Find(ref row, this.emailsToken); + r = LayoutType.TypedArray.ReadScope(ref row, ref root, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + while (childScope.MoveNext(ref row)) + { + r = LayoutType.Utf8.ReadSparse(ref row, ref childScope, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + } + + root.Skip(ref row, ref childScope); + root.Find(ref row, this.phoneNumbersToken); + r = LayoutType.TypedArray.ReadScope(ref row, ref root, out childScope); + if (r != Result.Success) + { + return r; + } + + while (childScope.MoveNext(ref row)) + { + r = LayoutType.Utf8.ReadSparse(ref row, ref childScope, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + } + + root.Skip(ref row, ref childScope); + root.Find(ref row, this.addressesToken); + r = LayoutType.TypedMap.ReadScope(ref row, ref root, out childScope); + if (r != Result.Success) + { + return r; + } + + while (childScope.MoveNext(ref row)) + { + r = LayoutType.TypedTuple.ReadScope(ref row, ref childScope, out RowCursor tupleScope); + if (r != Result.Success) + { + return r; + } + + if (!tupleScope.MoveNext(ref row)) + { + return Result.InvalidRow; + } + + r = LayoutType.Utf8.ReadSparse(ref row, ref tupleScope, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + if (!tupleScope.MoveNext(ref row)) + { + return Result.InvalidRow; + } + + r = LayoutType.ImmutableUDT.ReadScope(ref row, ref tupleScope, out RowCursor valueScope); + if (r != Result.Success) + { + return r; + } + + r = this.addressSerializer.ReadBuffer(ref row, ref valueScope); + if (r != Result.Success) + { + return r; + } + + tupleScope.Skip(ref row, ref valueScope); + childScope.Skip(ref row, ref tupleScope); + } + + root.Skip(ref row, ref childScope); + + return LayoutType.Utf8.ReadVariable(ref row, ref root, this.confirmNumber, out Utf8Span _); + } + } + + private sealed class HotelsHybridRowSerializer : HybridRowSerializer + { + private static readonly Utf8String HotelIdName = Utf8String.TranscodeUtf16("hotel_id"); + private static readonly Utf8String NameName = Utf8String.TranscodeUtf16("name"); + private static readonly Utf8String PhoneName = Utf8String.TranscodeUtf16("phone"); + private static readonly Utf8String AddressName = Utf8String.TranscodeUtf16("address"); + private readonly LayoutColumn hotelId; + private readonly LayoutColumn name; + private readonly LayoutColumn phone; + private readonly LayoutColumn address; + private readonly StringToken addressToken; + private readonly AddressHybridRowSerializer addressSerializer; + + public HotelsHybridRowSerializer(Layout layout, LayoutResolver resolver) + { + layout.TryFind(HotelsHybridRowSerializer.HotelIdName, out this.hotelId); + layout.TryFind(HotelsHybridRowSerializer.NameName, out this.name); + layout.TryFind(HotelsHybridRowSerializer.PhoneName, out this.phone); + layout.TryFind(HotelsHybridRowSerializer.AddressName, out this.address); + layout.Tokenizer.TryFindToken(this.address.Path, out this.addressToken); + this.addressSerializer = new AddressHybridRowSerializer(resolver.Resolve(this.address.TypeArgs.SchemaId), resolver); + } + + public override Result WriteBuffer(ref RowBuffer row, ref RowCursor root, Dictionary tableValue) + { + foreach ((Utf8String key, object value) in tableValue) + { + Result r; + switch (0) + { + case 0 when key.Equals(HotelsHybridRowSerializer.HotelIdName): + if (value != null) + { + r = LayoutType.Utf8.WriteFixed(ref row, ref root, this.hotelId, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(HotelsHybridRowSerializer.NameName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.name, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(HotelsHybridRowSerializer.PhoneName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.phone, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(HotelsHybridRowSerializer.AddressName): + if (value != null) + { + root.Find(ref row, this.addressToken); + r = LayoutType.UDT.WriteScope(ref row, ref root, this.address.TypeArgs, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + r = this.addressSerializer.WriteBuffer(ref row, ref childScope, (Dictionary)value); + if (r != Result.Success) + { + return r; + } + + root.Skip(ref row, ref childScope); + } + + break; + + default: + Contract.Fail($"Unknown property name: {key}"); + break; + } + } + + return Result.Success; + } + + public override Result ReadBuffer(ref RowBuffer row, ref RowCursor root) + { + Result r = LayoutType.Utf8.ReadFixed(ref row, ref root, this.hotelId, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.name, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.phone, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + root.Find(ref row, this.addressToken); + r = LayoutType.UDT.ReadScope(ref row, ref root, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + r = this.addressSerializer.ReadBuffer(ref row, ref childScope); + if (r != Result.Success) + { + return r; + } + + root.Skip(ref row, ref childScope); + return Result.Success; + } + } + + private sealed class RoomsHybridRowSerializer : HybridRowSerializer + { + private static readonly Utf8String HotelIdName = Utf8String.TranscodeUtf16("hotel_id"); + private static readonly Utf8String DateName = Utf8String.TranscodeUtf16("date"); + private static readonly Utf8String RoomNumberName = Utf8String.TranscodeUtf16("room_number"); + private static readonly Utf8String IsAvailableName = Utf8String.TranscodeUtf16("is_available"); + private readonly LayoutColumn hotelId; + private readonly LayoutColumn date; + private readonly LayoutColumn roomNumber; + private readonly LayoutColumn isAvailable; + + // ReSharper disable once UnusedParameter.Local + public RoomsHybridRowSerializer(Layout layout, LayoutResolver resolver) + { + layout.TryFind(RoomsHybridRowSerializer.HotelIdName, out this.hotelId); + layout.TryFind(RoomsHybridRowSerializer.DateName, out this.date); + layout.TryFind(RoomsHybridRowSerializer.RoomNumberName, out this.roomNumber); + layout.TryFind(RoomsHybridRowSerializer.IsAvailableName, out this.isAvailable); + } + + public override Result WriteBuffer(ref RowBuffer row, ref RowCursor root, Dictionary tableValue) + { + foreach ((Utf8String key, object value) in tableValue) + { + Result r; + switch (0) + { + case 0 when key.Equals(RoomsHybridRowSerializer.HotelIdName): + if (value != null) + { + r = LayoutType.Utf8.WriteFixed(ref row, ref root, this.hotelId, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(RoomsHybridRowSerializer.DateName): + if (value != null) + { + r = LayoutType.DateTime.WriteFixed(ref row, ref root, this.date, (DateTime)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(RoomsHybridRowSerializer.RoomNumberName): + if (value != null) + { + r = LayoutType.UInt8.WriteFixed(ref row, ref root, this.roomNumber, (byte)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(RoomsHybridRowSerializer.IsAvailableName): + if (value != null) + { + r = LayoutType.Boolean.WriteFixed(ref row, ref root, this.isAvailable, (bool)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + default: + Contract.Fail($"Unknown property name: {key}"); + break; + } + } + + return Result.Success; + } + + public override Result ReadBuffer(ref RowBuffer row, ref RowCursor root) + { + Result r = LayoutType.Utf8.ReadFixed(ref row, ref root, this.hotelId, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.DateTime.ReadFixed(ref row, ref root, this.date, out DateTime _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.UInt8.ReadFixed(ref row, ref root, this.roomNumber, out byte _); + if (r != Result.Success) + { + return r; + } + + return LayoutType.Boolean.ReadFixed(ref row, ref root, this.isAvailable, out bool _); + } + } + + private sealed class PostalCodeHybridRowSerializer : HybridRowSerializer + { + private static readonly Utf8String ZipName = Utf8String.TranscodeUtf16("zip"); + private static readonly Utf8String Plus4Name = Utf8String.TranscodeUtf16("plus4"); + private readonly LayoutColumn zip; + private readonly LayoutColumn plus4; + private readonly StringToken plus4Token; + + // ReSharper disable once UnusedParameter.Local + public PostalCodeHybridRowSerializer(Layout layout, LayoutResolver resolver) + { + layout.TryFind(PostalCodeHybridRowSerializer.ZipName, out this.zip); + layout.TryFind(PostalCodeHybridRowSerializer.Plus4Name, out this.plus4); + layout.Tokenizer.TryFindToken(this.plus4.Path, out this.plus4Token); + } + + public override Result WriteBuffer(ref RowBuffer row, ref RowCursor root, Dictionary tableValue) + { + foreach ((Utf8String key, object value) in tableValue) + { + Result r; + switch (0) + { + case 0 when key.Equals(PostalCodeHybridRowSerializer.ZipName): + if (value != null) + { + r = LayoutType.Int32.WriteFixed(ref row, ref root, this.zip, (int)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(PostalCodeHybridRowSerializer.Plus4Name): + if (value != null) + { + root.Find(ref row, this.plus4Token); + r = LayoutType.Int16.WriteSparse(ref row, ref root, (short)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + default: + Contract.Fail($"Unknown property name: {key}"); + break; + } + } + + return Result.Success; + } + + public override Result ReadBuffer(ref RowBuffer row, ref RowCursor root) + { + Result r = LayoutType.Int32.ReadFixed(ref row, ref root, this.zip, out int _); + if (r != Result.Success) + { + return r; + } + + root.Find(ref row, this.plus4Token); + return LayoutType.Int16.ReadSparse(ref row, ref root, out short _); + } + } + + private sealed class AddressHybridRowSerializer : HybridRowSerializer + { + private static readonly Utf8String StreetName = Utf8String.TranscodeUtf16("street"); + private static readonly Utf8String CityName = Utf8String.TranscodeUtf16("city"); + private static readonly Utf8String StateName = Utf8String.TranscodeUtf16("state"); + private static readonly Utf8String PostalCodeName = Utf8String.TranscodeUtf16("postal_code"); + private readonly LayoutColumn street; + private readonly LayoutColumn city; + private readonly LayoutColumn state; + private readonly LayoutColumn postalCode; + private readonly StringToken postalCodeToken; + private readonly PostalCodeHybridRowSerializer postalCodeSerializer; + + public AddressHybridRowSerializer(Layout layout, LayoutResolver resolver) + { + layout.TryFind(AddressHybridRowSerializer.StreetName, out this.street); + layout.TryFind(AddressHybridRowSerializer.CityName, out this.city); + layout.TryFind(AddressHybridRowSerializer.StateName, out this.state); + layout.TryFind(AddressHybridRowSerializer.PostalCodeName, out this.postalCode); + layout.Tokenizer.TryFindToken(this.postalCode.Path, out this.postalCodeToken); + this.postalCodeSerializer = new PostalCodeHybridRowSerializer(resolver.Resolve(this.postalCode.TypeArgs.SchemaId), resolver); + } + + public override Result WriteBuffer(ref RowBuffer row, ref RowCursor root, Dictionary tableValue) + { + foreach ((Utf8String key, object value) in tableValue) + { + Result r; + switch (0) + { + case 0 when key.Equals(AddressHybridRowSerializer.StreetName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.street, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(AddressHybridRowSerializer.CityName): + if (value != null) + { + r = LayoutType.Utf8.WriteVariable(ref row, ref root, this.city, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(AddressHybridRowSerializer.StateName): + if (value != null) + { + r = LayoutType.Utf8.WriteFixed(ref row, ref root, this.state, (Utf8String)value); + if (r != Result.Success) + { + return r; + } + } + + break; + + case 0 when key.Equals(AddressHybridRowSerializer.PostalCodeName): + if (value != null) + { + root.Find(ref row, this.postalCodeToken); + r = LayoutType.UDT.WriteScope(ref row, ref root, this.postalCode.TypeArgs, out RowCursor childScope); + + if (r != Result.Success) + { + return r; + } + + r = this.postalCodeSerializer.WriteBuffer(ref row, ref childScope, (Dictionary)value); + if (r != Result.Success) + { + return r; + } + + root.Skip(ref row, ref childScope); + } + + break; + + default: + Contract.Fail($"Unknown property name: {key}"); + break; + } + } + + return Result.Success; + } + + public override Result ReadBuffer(ref RowBuffer row, ref RowCursor root) + { + Result r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.street, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadVariable(ref row, ref root, this.city, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + r = LayoutType.Utf8.ReadFixed(ref row, ref root, this.state, out Utf8Span _); + if (r != Result.Success) + { + return r; + } + + root.Find(ref row, this.postalCodeToken); + r = LayoutType.UDT.ReadScope(ref row, ref root, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + r = this.postalCodeSerializer.ReadBuffer(ref row, ref childScope); + if (r != Result.Success) + { + return r; + } + + root.Skip(ref row, ref childScope); + return Result.Success; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/GenerateBenchmarkSuite.cs b/dotnet/src/HybridRow.Tests.Perf/GenerateBenchmarkSuite.cs new file mode 100644 index 0000000..fb34dd7 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/GenerateBenchmarkSuite.cs @@ -0,0 +1,100 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(TestData.SchemaFile, TestData.Target)] + public sealed class GenerateBenchmarkSuite : BenchmarkSuiteBase + { + private string sdl; + + [TestInitialize] + public void ParseNamespaceExample() + { + this.sdl = File.ReadAllText(TestData.SchemaFile); + Namespace schema = Namespace.Parse(this.sdl); + this.DefaultResolver = new LayoutResolverNamespace(schema, SystemSchema.LayoutResolver); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task GenerateHotelBenchmarkAsync() + { + await this.GenerateBenchmarkAsync("Hotels", 100, TestData.HotelExpected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task GenerateRoomsBenchmarkAsync() + { + await this.GenerateBenchmarkAsync("Available_Rooms_By_Hotel_Date", 100, TestData.RoomsExpected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task GenerateGuestsBenchmarkAsync() + { + await this.GenerateBenchmarkAsync("Guests", 50, TestData.GuestsExpected); + } + + private static List> GenerateBenchmarkInputs( + LayoutResolverNamespace resolver, + string schemaName, + int outerLoopIterations) + { + HybridRowGeneratorConfig generatorConfig = new HybridRowGeneratorConfig(); + const int seed = 42; + RandomGenerator rand = new RandomGenerator(new Random(seed)); + HybridRowValueGenerator valueGenerator = new HybridRowValueGenerator(rand, generatorConfig); + + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + List> rows = new List>(outerLoopIterations); + for (int iteration = 0; iteration != outerLoopIterations; iteration += outerLoopIterations < 0 ? 0 : 1) + { + TypeArgument typeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(layout.SchemaId)); + Dictionary rowValue = (Dictionary)valueGenerator.GenerateLayoutType(resolver, typeArg); + rows.Add(rowValue); + } + + return rows; + } + + private async Task GenerateBenchmarkAsync(string schemaName, int outerLoopIterations, string expectedFile) + { + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + List> + rows = GenerateBenchmarkSuite.GenerateBenchmarkInputs(resolver, schemaName, outerLoopIterations); + + Schema tableSchema = resolver.Namespace.Schemas.Find(x => x.Name == schemaName); + TypeArgument typeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(tableSchema.SchemaId)); + + bool allMatch = rows.Count == expected.Count; + for (int i = 0; allMatch && i < rows.Count; i++) + { + allMatch |= HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, expected[i], rows[i], typeArg); + } + + if (!allMatch) + { + await BenchmarkSuiteBase.WriteAllRowsAsync(expectedFile, this.sdl, resolver, resolver.Resolve(tableSchema.SchemaId), rows); + Console.WriteLine("Updated expected file at: {0}", Path.GetFullPath(expectedFile)); + Assert.IsTrue(allMatch, "Expected output does not match expected file."); + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/GenerateProtoBuf.cmd b/dotnet/src/HybridRow.Tests.Perf/GenerateProtoBuf.cmd new file mode 100644 index 0000000..7e2cdcd --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/GenerateProtoBuf.cmd @@ -0,0 +1,7 @@ +%NugetMachineInstallRoot%\google.protobuf.tools\3.5.1\tools\windows_x64\protoc.exe --csharp_out=CassandraProto TestData\CassandraHotelSchema.proto --proto_path=. --proto_path=%NugetMachineInstallRoot%\google.protobuf.tools\3.5.1\tools + +move CassandraProto\CassandraHotelSchema.cs CassandraProto\CassandraHotelSchema.tmp +echo #pragma warning disable DontUseNamespaceAliases // Namespace Aliases should be avoided > CassandraProto\CassandraHotelSchema.cs +echo #pragma warning disable NamespaceMatchesFolderStructure // Namespace Declarations must match folder structure >> CassandraProto\CassandraHotelSchema.cs +cat CassandraProto\CassandraHotelSchema.tmp >> CassandraProto\CassandraHotelSchema.cs +del CassandraProto\CassandraHotelSchema.tmp diff --git a/dotnet/src/HybridRow.Tests.Perf/HybridRowPerf.csv b/dotnet/src/HybridRow.Tests.Perf/HybridRowPerf.csv new file mode 100644 index 0000000..41041e2 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/HybridRowPerf.csv @@ -0,0 +1,43 @@ +RunId,Model,Operation,Schema,API,Iterations,Size (bytes),Total (ms),Duration (ms),Allocated (bytes),ThreadId,Gen0,Gen1,Gen2,Total Allocated (bytes) +636970252347090152,CodeGen,Write,Guests,Protobuf,50,1328,795.36630000,0.01590733,5084.00000000,8,40,0,0,254200600 +636970252347090152,CodeGen,Write,Hotels,Protobuf,100,156,2303.97720000,0.00230398,861.00000000,8,137,0,0,861760000 +636970252347090152,CodeGen,Write,Rooms,Protobuf,100,31,629.30810000,0.00062931,343.00000000,12,54,0,0,343440072 +636970252347090152,CodeGen,Read,Guests,Protobuf,50,1328,313.95220000,0.00627904,3556.00000000,12,28,0,0,177840000 +636970252347090152,CodeGen,Read,Hotels,Protobuf,100,156,795.37780000,0.00079538,485.00000000,13,77,0,0,485760072 +636970252347090152,CodeGen,Read,Rooms,Protobuf,100,31,252.23010000,0.00025223,183.00000000,8,29,0,0,183440000 +636970252347090152,CodeGen,Write,Guests,HybridRowGen,50,1222,1118.28520000,0.02236570,0.00000000,12,0,0,0,0 +636970252347090152,CodeGen,Write,Hotels,HybridRowGen,100,143,2044.81250000,0.00204481,0.00000000,8,0,0,0,0 +636970252347090152,CodeGen,Write,Rooms,HybridRowGen,100,23,534.58980000,0.00053459,0.00000000,8,0,0,0,0 +636970252347090152,CodeGen,Read,Guests,HybridRowGen,50,1222,207.97910000,0.00415958,168.00000000,12,1,0,0,8400000 +636970252347090152,CodeGen,Read,Hotels,HybridRowGen,100,143,731.67050000,0.00073167,0.00000000,12,0,0,0,0 +636970252347090152,CodeGen,Read,Rooms,HybridRowGen,100,23,184.29990000,0.00018430,0.00000000,12,0,0,0,0 +636970252347090152,Schematized,Write,Hotels,JSON,100,342,926.68970000,0.00926690,1653.00000000,13,26,1,0,165382696 +636970252347090152,Schematized,Write,Rooms,JSON,100,129,3728.27760000,0.00372828,871.00000000,12,138,0,0,871474472 +636970252347090152,Schematized,Write,Guests,JSON,50,2467,2236.45660000,0.04472913,6252.00000000,13,49,0,0,312604400 +636970252347090152,Schematized,Read,Hotels,JSON,100,342,283.91940000,0.00283919,8557.00000000,12,136,0,0,855776000 +636970252347090152,Schematized,Read,Rooms,JSON,100,129,1629.21600000,0.00162922,7846.00000000,13,1246,0,0,7846400000 +636970252347090152,Schematized,Read,Guests,JSON,50,2467,626.56730000,0.01253135,11421.00000000,12,90,0,0,571096000 +636970252347090152,Schematized,Write,Hotels,BSON,100,240,373.35650000,0.00373357,2125.00000000,13,33,0,0,212576000 +636970252347090152,Schematized,Write,Rooms,BSON,100,74,1215.03590000,0.00121504,815.00000000,13,129,0,0,815440000 +636970252347090152,Schematized,Write,Guests,BSON,50,1782,1018.99680000,0.02037994,9041.00000000,12,71,0,0,452072000 +636970252347090152,Schematized,Read,Hotels,BSON,100,240,199.81420000,0.00199814,2688.00000000,13,42,0,0,268800000 +636970252347090152,Schematized,Read,Rooms,BSON,100,74,888.62140000,0.00088862,1408.00000000,6,223,0,0,1408000000 +636970252347090152,Schematized,Read,Guests,BSON,50,1782,431.03020000,0.00862060,9146.00000000,13,72,0,0,457344000 +636970252347090152,Schematized,Write,Hotels,Layout,100,143,264.21190000,0.00264212,0.00000000,6,0,0,0,0 +636970252347090152,Schematized,Write,Rooms,Layout,100,23,64.51090000,0.00064511,0.00000000,6,0,0,0,0 +636970252347090152,Schematized,Write,Guests,Layout,50,1222,1221.28360000,0.02442567,0.00000000,12,0,0,0,0 +636970252347090152,Schematized,Read,Hotels,Layout,100,143,79.88940000,0.00079889,0.00000000,13,0,0,0,0 +636970252347090152,Schematized,Read,Rooms,Layout,100,23,21.45470000,0.00021455,0.00000000,12,0,0,0,0 +636970252347090152,Schematized,Read,Guests,Layout,50,1222,248.11960000,0.00496239,168.00000000,6,1,0,0,8400000 +636970252347090152,Schematized,Write,Hotels,HybridRow,100,143,352.73410000,0.00352734,0.00000000,12,0,0,0,0 +636970252347090152,Schematized,Write,Rooms,HybridRow,100,23,1129.17010000,0.00112917,0.00000000,6,0,0,0,0 +636970252347090152,Schematized,Write,Guests,HybridRow,50,1222,1418.34740000,0.02836695,0.00000000,12,0,0,0,0 +636970252347090152,Schematized,Read,Hotels,HybridRow,100,143,107.00560000,0.00107006,0.00000000,6,0,0,0,0 +636970252347090152,Schematized,Read,Rooms,HybridRow,100,23,307.03740000,0.00030704,0.00000000,12,0,0,0,0 +636970252347090152,Schematized,Read,Guests,HybridRow,50,1222,356.78610000,0.00713572,168.00000000,6,1,0,0,8400000 +636970252347090152,Unschematized,Write,Messages1K,HybridRowSparse,1001,3405,312.43510000,0.03121230,0.00000000,13,0,0,0,0 +636970252347090152,Unschematized,Read,Messages1K,HybridRowSparse,1001,3405,153.64990000,0.01534964,0.00000000,12,0,0,0,0 +636970252347090152,Unschematized,Write,Messages1K,BSON,1001,3949,301.69900000,0.03013976,30476.00000000,13,48,0,0,305065120 +636970252347090152,Unschematized,Read,Messages1K,BSON,1001,3949,177.64080000,0.01774633,23166.00000000,8,36,0,0,231892392 +636970252347090152,Unschematized,Write,Messages1K,JSON,1001,5693,999.48440000,0.09984859,16900.00000000,8,26,0,0,169178696 +636970252347090152,Unschematized,Read,Messages1K,JSON,1001,5693,291.96320000,0.02916715,32040.00000000,8,49,1,1,320720528 diff --git a/dotnet/src/HybridRow.Tests.Perf/JsonModelRowGenerator.cs b/dotnet/src/HybridRow.Tests.Perf/JsonModelRowGenerator.cs new file mode 100644 index 0000000..0c99a59 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/JsonModelRowGenerator.cs @@ -0,0 +1,132 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public ref struct JsonModelRowGenerator + { + private RowBuffer row; + + public JsonModelRowGenerator(int capacity, Layout layout, LayoutResolver resolver, ISpanResizer resizer = default) + { + this.row = new RowBuffer(capacity, resizer); + this.row.InitLayout(HybridRowVersion.V1, layout, resolver); + } + + public int Length => this.row.Length; + + public byte[] ToArray() => this.row.ToArray(); + + public void WriteTo(Stream stream) + { + this.row.WriteTo(stream); + } + + public bool ReadFrom(Stream stream, int length) + { + return this.row.ReadFrom(stream, length, HybridRowVersion.V1, this.row.Resolver); + } + + public void Reset() + { + Layout layout = this.row.Resolver.Resolve(this.row.Header.SchemaId); + this.row.InitLayout(HybridRowVersion.V1, layout, this.row.Resolver); + } + + public RowReader GetReader() + { + return new RowReader(ref this.row); + } + + public Result WriteBuffer(Dictionary value) + { + return RowWriter.WriteBuffer( + ref this.row, + value, + (ref RowWriter writer, TypeArgument typeArg, Dictionary dict) => + { + foreach ((Utf8String propPath, object propValue) in dict) + { + Result result = JsonModelRowGenerator.JsonModelSwitch(ref writer, propPath, propValue); + if (result != Result.Success) + { + return result; + } + } + + return Result.Success; + }); + } + + private static Result JsonModelSwitch(ref RowWriter writer, Utf8String path, object value) + { + switch (value) + { + case null: + return writer.WriteNull(path); + case bool x: + return writer.WriteBool(path, x); + case long x: + return writer.WriteInt64(path, x); + case double x: + return writer.WriteFloat64(path, x); + case string x: + return writer.WriteString(path, x); + case Utf8String x: + return writer.WriteString(path, x.Span); + case byte[] x: + return writer.WriteBinary(path, x); + case ReadOnlyMemory x: + return writer.WriteBinary(path, x.Span); + case Dictionary x: + return writer.WriteScope( + path, + new TypeArgument(LayoutType.Object), + x, + (ref RowWriter writer2, TypeArgument typeArg, Dictionary dict) => + { + foreach ((Utf8String propPath, object propValue) in dict) + { + Result result = JsonModelRowGenerator.JsonModelSwitch(ref writer2, propPath, propValue); + if (result != Result.Success) + { + return result; + } + } + + return Result.Success; + }); + case List x: + return writer.WriteScope( + path, + new TypeArgument(LayoutType.Array), + x, + (ref RowWriter writer2, TypeArgument typeArg, List list) => + { + foreach (object elm in list) + { + Result result = JsonModelRowGenerator.JsonModelSwitch(ref writer2, null, elm); + if (result != Result.Success) + { + return result; + } + } + + return Result.Success; + }); + default: + Contract.Assert(false, $"Unknown type will be ignored: {value.GetType().Name}"); + return Result.Failure; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/Measurements.cs b/dotnet/src/HybridRow.Tests.Perf/Measurements.cs new file mode 100644 index 0000000..e0fb8b3 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/Measurements.cs @@ -0,0 +1,89 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.IO; + using System.Text; + + internal class Measurements : IDisposable + { + private static readonly long RunId = DateTime.UtcNow.Ticks; + private readonly FileStream file; + private readonly TextWriter writer; + + public Measurements(string path) + { + FileInfo info = new FileInfo(path); + if (info.Exists) + { + this.file = new FileStream(path, FileMode.Append); + this.writer = new StreamWriter(this.file, Encoding.ASCII); + } + else + { + this.file = new FileStream(path, FileMode.CreateNew); + this.writer = new StreamWriter(this.file, Encoding.ASCII); + this.writer.WriteLine( + "RunId,Model,Operation,Schema,API,Iterations,Size (bytes),Total (ms),Duration (ms),Allocated (bytes),ThreadId,Gen0,Gen1,Gen2,Total Allocated (bytes)"); + } + } + + public void Dispose() + { + this.writer.Flush(); + this.writer.Dispose(); + this.file.Dispose(); + } + + public void WriteMeasurement(string model, string operation, string schema, string api, + int outerLoopIterations, int innerLoopIterations, long totalSize, double totalDurationMs, + int threadId, int gen0, int gen1, int gen2, long totalAllocatedBytes) + { + Console.WriteLine( + "RunId: {0}, \nModel: {1} \nOperation: {2} \nSchema: {3} \nAPI: {4}", + Measurements.RunId, + model, + operation, + schema, + api); + + Console.WriteLine( + "\n\nIterations: {0} \nSize (bytes): {1:F0} \nTotal (ms): {2:F4} \nDuration (ms): {3:F4} \nAllocated (bytes): {4:F4}", + outerLoopIterations, + totalSize / outerLoopIterations, + totalDurationMs, + totalDurationMs / (outerLoopIterations * innerLoopIterations), + totalAllocatedBytes / (outerLoopIterations * innerLoopIterations)); + + Console.WriteLine( + "\n\nThread: {0} \nCollections: {1}, {2}, {3} \nTotal Allocated: {4:n0} (bytes)", + threadId, + gen0, + gen1, + gen2, + totalAllocatedBytes); + + + this.writer.WriteLine( + "{0},{1},{2},{3},{4},{5},{6:F0},{7:F8},{8:F8},{9:F8},{10},{11},{12},{13},{14:0}", + Measurements.RunId, + model, + operation, + schema, + api, + outerLoopIterations, + totalSize / outerLoopIterations, + totalDurationMs, + totalDurationMs / (outerLoopIterations * innerLoopIterations), + totalAllocatedBytes / (outerLoopIterations * innerLoopIterations), + threadId, + gen0, + gen1, + gen2, + totalAllocatedBytes); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/MicroBenchmarkSuiteBase.cs b/dotnet/src/HybridRow.Tests.Perf/MicroBenchmarkSuiteBase.cs new file mode 100644 index 0000000..56f69cd --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/MicroBenchmarkSuiteBase.cs @@ -0,0 +1,128 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.CompilerServices; + using System.Threading; + using JetBrains.Profiler.Api; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + + [SuppressMessage("Microsoft.Reliability", "CA2001:Avoid calling problematic methods", Justification = "Perf Benchmark")] + public class MicroBenchmarkSuiteBase : BenchmarkSuiteBase + { + private const int WarmCount = 5; + private const string MetricsResultFile = "HybridRowPerf.csv"; + + [SuppressMessage("Microsoft.Reliability", "CA2001:Avoid calling problematic methods", Justification = "Perf Benchmark")] + private protected static void Benchmark( + string model, + string operation, + string schema, + string api, + int innerLoopIterations, + ref BenchmarkContext context, + BenchmarkBody loopBody, + BenchmarkMeasure measure, + List expected) + { + Stopwatch sw = new Stopwatch(); + double durationMs = 0; + long rowSize = 0; + + // Warm + int warm = Math.Min(MicroBenchmarkSuiteBase.WarmCount, expected.Count); + for (int i = 0; i < warm; i++) + { + for (int innerLoop = 0; innerLoop < innerLoopIterations; innerLoop++) + { + loopBody(ref context, expected[i]); + } + } + + // Execute + GC.Collect(); + GC.WaitForPendingFinalizers(); + Thread.Sleep(1000); + int gen0 = GC.CollectionCount(0); + int gen1 = GC.CollectionCount(1); + int gen2 = GC.CollectionCount(2); + long allocated = GC.GetAllocatedBytesForCurrentThread(); + int threadId = Thread.CurrentThread.ManagedThreadId; + ThreadPriority currentPriority = Thread.CurrentThread.Priority; + Thread.CurrentThread.Priority = ThreadPriority.Highest; + MemoryProfiler.CollectAllocations(true); + MemoryProfiler.GetSnapshot(); + try + { + foreach (TValue tableValue in expected) + { + sw.Restart(); + MicroBenchmarkSuiteBase.BenchmarkInnerLoop(innerLoopIterations, tableValue, ref context, loopBody); + sw.Stop(); + durationMs += sw.Elapsed.TotalMilliseconds; + rowSize += measure(ref context, tableValue); + } + } + finally + { + Thread.CurrentThread.Priority = currentPriority; + gen0 = GC.CollectionCount(0) - gen0; + gen1 = GC.CollectionCount(1) - gen1; + gen2 = GC.CollectionCount(2) - gen2; + allocated = GC.GetAllocatedBytesForCurrentThread() - allocated; + MemoryProfiler.GetSnapshot(); + MemoryProfiler.CollectAllocations(false); + } + + using (Measurements m = new Measurements(MicroBenchmarkSuiteBase.MetricsResultFile)) + { + m.WriteMeasurement( + model: model, + operation: operation, + schema: schema, + api: api, + outerLoopIterations: expected.Count, + innerLoopIterations: innerLoopIterations, + totalSize: rowSize, + totalDurationMs: durationMs, + threadId: threadId, + gen0: gen0, + gen1: gen1, + gen2: gen2, + totalAllocatedBytes: allocated); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void BenchmarkInnerLoop( + int innerLoopIterations, + TValue tableValue, + ref BenchmarkContext context, + BenchmarkBody loopBody) + { + for (int innerLoop = 0; innerLoop < innerLoopIterations; innerLoop++) + { + loopBody(ref context, tableValue); + } + } + + private protected ref struct BenchmarkContext + { + public CodeGenRowGenerator CodeGenWriter; + public ProtobufRowGenerator ProtobufWriter; + public WriteRowGenerator PatchWriter; + public StreamingRowGenerator StreamingWriter; + public JsonModelRowGenerator JsonModelWriter; + } + + private protected delegate void BenchmarkBody(ref BenchmarkContext context, TValue value); + + private protected delegate long BenchmarkMeasure(ref BenchmarkContext context, TValue value); + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.csproj b/dotnet/src/HybridRow.Tests.Perf/Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.csproj new file mode 100644 index 0000000..34e4c7d --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.csproj @@ -0,0 +1,52 @@ + + + + true + true + {26A73F4A-AC9E-46AF-9445-286EE9EDA3EE} + Library + Test + Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf + Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf + netcoreapp2.2 + True + AnyCPU + + + + MsTest_Latest + FrameworkCore20 + X64 + $(OutDir) + + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + \ No newline at end of file diff --git a/dotnet/src/HybridRow.Tests.Perf/Properties/AssemblyInfo.cs b/dotnet/src/HybridRow.Tests.Perf/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..9421396 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("26A73F4A-AC9E-46AF-9445-286EE9EDA3EE")] diff --git a/dotnet/src/HybridRow.Tests.Perf/ProtobufRowGenerator.cs b/dotnet/src/HybridRow.Tests.Perf/ProtobufRowGenerator.cs new file mode 100644 index 0000000..e6aa1ea --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/ProtobufRowGenerator.cs @@ -0,0 +1,315 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using Google.Protobuf; + using Google.Protobuf.Collections; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.VisualStudio.TestTools.UnitTesting; +#pragma warning disable DontUseNamespaceAliases // Namespace Aliases should be avoided + using pb = Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf; + +#pragma warning restore DontUseNamespaceAliases // Namespace Aliases should be avoided + + internal ref struct ProtobufRowGenerator + { + private readonly string schemaName; + private readonly byte[] buffer; + private ReadOnlySpan active; + + public ProtobufRowGenerator(string schemaName, int capacity) + { + this.schemaName = schemaName; + this.buffer = new byte[capacity]; + this.active = this.buffer; + } + + public void WriteBuffer(Dictionary tableValue) + { + switch (this.schemaName) + { + case "Hotels": + this.WriteBufferHotel(tableValue); + break; + + case "Guests": + this.WriteBufferGuest(tableValue); + break; + + case "Available_Rooms_By_Hotel_Date": + this.WriteBufferRoom(tableValue); + break; + + default: + Contract.Fail($"Unknown schema will be ignored: {this.schemaName}"); + break; + } + } + + public void ReadBuffer(byte[] buffer) + { + switch (this.schemaName) + { + case "Hotels": + ProtobufRowGenerator.ReadBufferHotel(buffer); + break; + + case "Guests": + ProtobufRowGenerator.ReadBufferGuest(buffer); + break; + + case "Available_Rooms_By_Hotel_Date": + ProtobufRowGenerator.ReadBufferRoom(buffer); + break; + + default: + Contract.Fail($"Unknown schema will be ignored: {this.schemaName}"); + break; + } + } + + public int Length => this.active.Length; + + public byte[] ToArray() + { + return this.active.ToArray(); + } + + private void WriteBufferGuest(Dictionary tableValue) + { + pb.Guests room = new pb.Guests(); + using (CodedOutputStream stm = new CodedOutputStream(this.buffer)) + { + foreach ((Utf8String key, object value) in tableValue) + { + switch (key.ToString()) + { + case "guest_id": + room.GuestId = ((Guid?)value)?.ToString(); + break; + + case "first_name": + room.FirstName = ((Utf8String)value)?.ToString(); + break; + + case "last_name": + room.LastName = ((Utf8String)value)?.ToString(); + break; + + case "title": + room.Title = ((Utf8String)value)?.ToString(); + break; + + case "emails": + if (value != null) + { + ProtobufRowGenerator.PopulateStringList(room.Emails, (List)value); + } + + break; + + case "phone_numbers": + if (value != null) + { + ProtobufRowGenerator.PopulateStringList(room.PhoneNumbers, (List)value); + } + + break; + + case "addresses": + if (value != null) + { + ProtobufRowGenerator.PopulateStringAddressMap(room.Addresses, (List)value); + } + + break; + + case "confirm_number": + room.ConfirmNumber = ((Utf8String)value)?.ToString(); + break; + + default: + Assert.Fail("should never happen"); + break; + } + } + + room.WriteTo(stm); + stm.Flush(); + this.active = this.buffer.AsSpan(0, (int)stm.Position); + } + } + + private void WriteBufferHotel(Dictionary tableValue) + { + pb.Hotels room = new pb.Hotels(); + using (CodedOutputStream stm = new CodedOutputStream(this.buffer)) + { + foreach ((Utf8String key, object value) in tableValue) + { + switch (key.ToString()) + { + case "hotel_id": + room.HotelId = ((Utf8String)value)?.ToString(); + break; + + case "name": + room.Name = ((Utf8String)value)?.ToString(); + break; + + case "phone": + room.Phone = ((Utf8String)value)?.ToString(); + break; + + case "address": + room.Address = value == null ? null : ProtobufRowGenerator.MakeAddress((Dictionary)value); + break; + + default: + Assert.Fail("should never happen"); + break; + } + } + + room.WriteTo(stm); + stm.Flush(); + this.active = this.buffer.AsSpan(0, (int)stm.Position); + } + } + + private void WriteBufferRoom(Dictionary tableValue) + { + pb.Available_Rooms_By_Hotel_Date room = new pb.Available_Rooms_By_Hotel_Date(); + using (CodedOutputStream stm = new CodedOutputStream(this.buffer)) + { + foreach ((Utf8String key, object value) in tableValue) + { + switch (key.ToString()) + { + case "hotel_id": + room.HotelId = ((Utf8String)value)?.ToString(); + break; + + case "date": + room.Date = ((DateTime?)value)?.Ticks; + break; + + case "room_number": + room.RoomNumber = (byte?)value; + break; + + case "is_available": + room.IsAvailable = (bool?)value; + break; + + default: + Assert.Fail("should never happen"); + break; + } + } + + room.WriteTo(stm); + stm.Flush(); + this.active = this.buffer.AsSpan(0, (int)stm.Position); + } + } + + private static void ReadBufferGuest(byte[] buffer) + { + pb.Guests item = new pb.Guests(); + item.MergeFrom(buffer); + } + + private static void ReadBufferHotel(byte[] buffer) + { + pb.Hotels item = new pb.Hotels(); + item.MergeFrom(buffer); + } + + private static void ReadBufferRoom(byte[] buffer) + { + pb.Available_Rooms_By_Hotel_Date item = new pb.Available_Rooms_By_Hotel_Date(); + item.MergeFrom(buffer); + } + + private static void PopulateStringList(RepeatedField field, List list) + { + foreach (object item in list) + { + field.Add(((Utf8String)item).ToString()); + } + } + + private static void PopulateStringAddressMap(MapField field, List list) + { + foreach (object item in list) + { + List tuple = (List)item; + string key = ((Utf8String)tuple[0]).ToString(); + pb.Address value = ProtobufRowGenerator.MakeAddress((Dictionary)tuple[1]); + field.Add(key, value); + } + } + + private static pb.PostalCode MakePostalCode(Dictionary tableValue) + { + pb.PostalCode postalCode = new pb.PostalCode(); + foreach ((Utf8String key, object value) in tableValue) + { + switch (key.ToString()) + { + case "zip": + postalCode.Zip = (int?)value; + break; + + case "plus4": + postalCode.Plus4 = (short?)value; + break; + + default: + Assert.Fail("should never happen"); + break; + } + } + + return postalCode; + } + + private static pb.Address MakeAddress(Dictionary tableValue) + { + pb.Address address = new pb.Address(); + foreach ((Utf8String key, object value) in tableValue) + { + switch (key.ToString()) + { + case "street": + address.Street = ((Utf8String)value)?.ToString(); + break; + + case "city": + address.City = ((Utf8String)value)?.ToString(); + break; + + case "state": + address.State = ((Utf8String)value)?.ToString(); + break; + + case "postal_code": + address.PostalCode = value == null ? null : ProtobufRowGenerator.MakePostalCode((Dictionary)value); + break; + + default: + Assert.Fail("should never happen"); + break; + } + } + + return address; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/ReaderBenchmark.cs b/dotnet/src/HybridRow.Tests.Perf/ReaderBenchmark.cs new file mode 100644 index 0000000..108a82e --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/ReaderBenchmark.cs @@ -0,0 +1,282 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [SuppressMessage("Microsoft.Reliability", "CA2001:Avoid calling problematic methods", Justification = "Perf Benchmark")] + [DeploymentItem(@"TestData\*.hr", "TestData")] + public sealed class ReaderBenchmark + { + private const int InitialCapacity = 2 * 1024 * 1024; + private const int WarmCount = 5; + private const int MeasureCount = 10; + private const string CombinedScriptsData = @"TestData\CombinedScriptsData.hr"; + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(ReaderBenchmark.CombinedScriptsData, "TestData")] + public async Task RowReaderAsync() + { + using (BenchmarkContext context = new BenchmarkContext(ReaderBenchmark.CombinedScriptsData, true, true)) + { + await ReaderBenchmark.BenchmarkAsync(context, ReaderBenchmark.RowReaderBenchmarkAsync); + } + } + + [TestMethod] + [Owner("jthunter")] + [Ignore] + public async Task SpecificFileAsync() + { + const string filename = @"E:\TestData\HybridRow\Lastfm.hr"; + using (BenchmarkContext context = new BenchmarkContext(filename, true, true)) + { + await ReaderBenchmark.BenchmarkAsync(context, ReaderBenchmark.RowReaderBenchmarkAsync); + } + } + + [TestMethod] + [Owner("jthunter")] + [Ignore] + public async Task AllAsync() + { + const string dir = @"E:\TestData\HybridRow"; + foreach (FileInfo childFile in new DirectoryInfo(dir).EnumerateFiles(@"*.hr")) + { + using (BenchmarkContext context = new BenchmarkContext(childFile.FullName, false, false)) + { + await ReaderBenchmark.BenchmarkAsync(context, ReaderBenchmark.RowReaderBenchmarkAsync); + } + } + } + + private static async Task RowReaderBenchmarkAsync(object ctx) + { + BenchmarkContext context = (BenchmarkContext)ctx; + MemorySpanResizer resizer = new MemorySpanResizer(ReaderBenchmark.InitialCapacity); + Result r = await context.Input.ReadRecordIOAsync( + record => + { + context.IncrementRecordCount(); + r = ReaderBenchmark.VisitOneRow(record, context.Resolver); + Assert.AreEqual(Result.Success, r); + return Result.Success; + }, + segment => + { + r = SegmentSerializer.Read(segment.Span, context.Resolver, out Segment _); + Assert.AreEqual(Result.Success, r); + + // TODO: do something with embedded schema. + return Result.Success; + }, + resizer); + + Assert.AreEqual(Result.Success, r); + } + + private static Result VisitOneRow(Memory buffer, LayoutResolver resolver) + { + RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, resolver); + RowReader reader = new RowReader(ref row); + return reader.VisitReader(); + } + + [SuppressMessage("Microsoft.Reliability", "CA2001:Avoid calling problematic methods", Justification = "Perf Benchmark")] + private static async Task BenchmarkAsync(BenchmarkContext context, Func body) + { + using (SingleThreadedTaskScheduler scheduler = new SingleThreadedTaskScheduler()) + { + // Warm + System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); + for (int i = 0; i < ReaderBenchmark.WarmCount; i++) + { + context.Reset(); + sw.Restart(); + await Task.Factory.StartNew(body, context, CancellationToken.None, TaskCreationOptions.None, scheduler).Unwrap(); + sw.Stop(); + if (context.ShowWarmSummary) + { + context.Summarize(sw.Elapsed); + } + } + + // Execute + double[] timing = new double[ReaderBenchmark.MeasureCount]; + for (int i = 0; i < ReaderBenchmark.MeasureCount; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + context.Reset(); + sw.Restart(); + await Task.Factory.StartNew(body, context, CancellationToken.None, TaskCreationOptions.None, scheduler).Unwrap(); + sw.Stop(); + if (context.ShowSummary) + { + context.Summarize(sw.Elapsed); + } + + timing[i] = sw.Elapsed.TotalMilliseconds; + } + + Array.Sort(timing); + Console.WriteLine( + $"File: {Path.GetFileNameWithoutExtension(context.InputFile)}, Mean: {timing[ReaderBenchmark.MeasureCount / 2]:F4}"); + } + } + + private sealed class BenchmarkContext : IDisposable + { + private readonly string inputFile; + private readonly bool showSummary; + private readonly bool showWarmSummary; + private long recordCount; + private readonly Stream input; + private readonly LayoutResolver resolver; + + public BenchmarkContext(string inputFile, bool showSummary = true, bool showWarmSummary = false) + { + this.inputFile = inputFile; + this.showSummary = showSummary; + this.showWarmSummary = showWarmSummary; + this.input = new FileStream(inputFile, FileMode.Open); + this.resolver = SystemSchema.LayoutResolver; + } + + public bool ShowSummary => this.showSummary; + + public bool ShowWarmSummary => this.showWarmSummary; + + public string InputFile => this.inputFile; + + public Stream Input => this.input; + + public LayoutResolver Resolver => this.resolver; + + public void IncrementRecordCount() + { + this.recordCount++; + } + + public void Reset() + { + this.recordCount = 0; + this.input.Seek(0, SeekOrigin.Begin); + } + + public void Summarize(TimeSpan duration) + { + Console.Write($"Total Time: {duration.TotalMilliseconds:F4}, "); + Console.WriteLine($"Record Count: {this.recordCount}"); + } + + public void Dispose() + { + this.input.Dispose(); + } + } + + private sealed class SingleThreadedTaskScheduler : TaskScheduler, IDisposable + { + private readonly Thread worker; + private readonly EventWaitHandle ready; + private readonly ConcurrentQueue tasks; + private readonly CancellationTokenSource cancel; + + // Creates a new instance with the specified degree of parallelism. + public SingleThreadedTaskScheduler() + { + this.tasks = new ConcurrentQueue(); + this.ready = new ManualResetEvent(false); + this.worker = new Thread(this.DoWork); + this.cancel = new CancellationTokenSource(); + this.worker.Start(); + } + + // Gets the maximum concurrency level supported by this scheduler. + public override int MaximumConcurrencyLevel => 1; + + public void Dispose() + { + if (!this.cancel.IsCancellationRequested) + { + this.cancel.Cancel(); + this.worker.Join(); + this.ready?.Dispose(); + this.cancel?.Dispose(); + } + } + + // Queues a task to the scheduler. + protected override void QueueTask(Task task) + { + lock (this.tasks) + { + this.tasks.Enqueue(task); + if (Thread.CurrentThread != this.worker) + { + this.ready.Set(); + } + } + } + + // Attempts to execute the specified task on the current thread. + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) + { + // If this thread isn't already processing a task, we don't support inlining + if (Thread.CurrentThread != this.worker) + { + return false; + } + + // If the task was previously queued, then skip it. + if (taskWasPreviouslyQueued) + { + return false; + } + + return this.TryExecuteTask(task); + } + + protected override bool TryDequeue(Task task) + { + return false; + } + + protected override IEnumerable GetScheduledTasks() + { + return null; + } + + private void DoWork() + { + while (!this.cancel.IsCancellationRequested) + { + if (this.tasks.TryDequeue(out Task item)) + { + this.TryExecuteTask(item); + } + else + { + this.ready.WaitOne(TimeSpan.FromSeconds(1)); + } + } + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/RowReaderExtensions.cs b/dotnet/src/HybridRow.Tests.Perf/RowReaderExtensions.cs new file mode 100644 index 0000000..9a34447 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/RowReaderExtensions.cs @@ -0,0 +1,97 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + internal static class RowReaderExtensions + { + public static Result VisitReader(this ref RowReader reader) + { + while (reader.Read()) + { + Utf8Span path = reader.PathSpan; + switch (reader.Type.LayoutCode) + { + case LayoutCode.Null: + case LayoutCode.Boolean: + case LayoutCode.Int8: + case LayoutCode.Int16: + case LayoutCode.Int32: + case LayoutCode.Int64: + case LayoutCode.UInt8: + case LayoutCode.UInt16: + case LayoutCode.UInt32: + case LayoutCode.UInt64: + case LayoutCode.VarInt: + case LayoutCode.VarUInt: + case LayoutCode.Float32: + case LayoutCode.Float64: + case LayoutCode.Float128: + case LayoutCode.Decimal: + case LayoutCode.DateTime: + case LayoutCode.UnixDateTime: + case LayoutCode.Guid: + case LayoutCode.MongoDbObjectId: + case LayoutCode.Utf8: + case LayoutCode.Binary: + break; + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + { + if (!reader.HasValue) + { + break; + } + + goto case LayoutCode.TypedTupleScope; + } + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + case LayoutCode.ArrayScope: + case LayoutCode.ImmutableArrayScope: + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + { + Result r = reader.ReadScope(null, (ref RowReader child, object _) => child.VisitReader()); + if (r != Result.Success) + { + return r; + } + + break; + } + + default: + { + Contract.Assert(false, $"Unknown type will be ignored: {reader.Type.LayoutCode}"); + break; + } + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/SchematizedMicroBenchmarkSuite.cs b/dotnet/src/HybridRow.Tests.Perf/SchematizedMicroBenchmarkSuite.cs new file mode 100644 index 0000000..e372b28 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/SchematizedMicroBenchmarkSuite.cs @@ -0,0 +1,583 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Internal; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using MongoDB.Bson.IO; + using Newtonsoft.Json; + + [TestClass] + [DeploymentItem(TestData.SchemaFile, TestData.Target)] + public sealed class SchematizedMicroBenchmarkSuite : MicroBenchmarkSuiteBase + { + private static readonly JsonSerializerSettings JsonSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented + }; + + private string sdl; + + [TestInitialize] + public void ParseNamespaceExample() + { + this.sdl = File.ReadAllText(TestData.SchemaFile); + Namespace schema = Namespace.Parse(this.sdl); + this.DefaultResolver = new LayoutResolverNamespace(schema, SystemSchema.LayoutResolver); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task JsonHotelWriteBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.JsonWriteBenchmark("Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task JsonRoomsWriteBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.JsonWriteBenchmark("Rooms", 10000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task JsonGuestsWriteBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.JsonWriteBenchmark("Guests", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task JsonHotelReadBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.JsonReadBenchmark("Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task JsonRoomsReadBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.JsonReadBenchmark("Rooms", 10000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task JsonGuestsReadBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace _) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.JsonReadBenchmark("Guests", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task BsonHotelWriteBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.BsonWriteBenchmark(resolver, "Hotels", "Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task BsonRoomsWriteBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.BsonWriteBenchmark(resolver, "Available_Rooms_By_Hotel_Date", "Rooms", 10000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task BsonGuestsWriteBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.BsonWriteBenchmark(resolver, "Guests", "Guests", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task BsonHotelReadBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.BsonReadBenchmark(resolver, "Hotels", "Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task BsonRoomsReadBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.BsonReadBenchmark(resolver, "Available_Rooms_By_Hotel_Date", "Rooms", 10000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task BsonGuestsReadBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.BsonReadBenchmark(resolver, "Guests", "Guests", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task LayoutHotelWriteBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.LayoutWriteBenchmark(resolver, "Hotels", "Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task LayoutRoomsWriteBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.LayoutWriteBenchmark(resolver, "Available_Rooms_By_Hotel_Date", "Rooms", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task LayoutGuestsWriteBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 1000; +#endif + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.LayoutWriteBenchmark(resolver, "Guests", "Guests", innerLoopIterations, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task LayoutHotelReadBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.LayoutReadBenchmark(resolver, "Hotels", "Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task LayoutRoomsReadBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.LayoutReadBenchmark(resolver, "Available_Rooms_By_Hotel_Date", "Rooms", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task LayoutGuestsReadBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 1000; +#endif + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.LayoutReadBenchmark(resolver, "Guests", "Guests", innerLoopIterations, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task StreamingHotelWriteBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.StreamingWriteBenchmark(resolver, "Hotels", "Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task StreamingRoomsWriteBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.StreamingWriteBenchmark(resolver, "Available_Rooms_By_Hotel_Date", "Rooms", 10000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task StreamingGuestsWriteBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.StreamingWriteBenchmark(resolver, "Guests", "Guests", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.HotelExpected, TestData.Target)] + public async Task StreamingHotelReadBenchmarkAsync() + { + string expectedFile = TestData.HotelExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.StreamingReadBenchmark(resolver, "Hotels", "Hotels", 1000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.RoomsExpected, TestData.Target)] + public async Task StreamingRoomsReadBenchmarkAsync() + { + string expectedFile = TestData.RoomsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.StreamingReadBenchmark(resolver, "Available_Rooms_By_Hotel_Date", "Rooms", 10000, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.GuestsExpected, TestData.Target)] + public async Task StreamingGuestsReadBenchmarkAsync() + { + string expectedFile = TestData.GuestsExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + SchematizedMicroBenchmarkSuite.StreamingReadBenchmark(resolver, "Guests", "Guests", 1000, expected); + } + + private static void JsonWriteBenchmark( + string dataSetName, + int innerLoopIterations, + List> expected) + { + Encoding utf8Encoding = new UTF8Encoding(); + JsonSerializer jsonSerializer = JsonSerializer.Create(SchematizedMicroBenchmarkSuite.JsonSettings); + using (MemoryStream jsonStream = new MemoryStream(BenchmarkSuiteBase.InitialCapacity)) + using (StreamWriter textWriter = new StreamWriter(jsonStream, utf8Encoding)) + using (JsonTextWriter jsonWriter = new JsonTextWriter(textWriter)) + { + BenchmarkContext ignoredContext = default; + jsonSerializer.Converters.Add(new Utf8StringJsonConverter()); + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Write", + dataSetName, + "JSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, Dictionary tableValue) => + { + jsonStream.SetLength(0); + jsonSerializer.Serialize(jsonWriter, tableValue); + jsonWriter.Flush(); + }, + (ref BenchmarkContext _, Dictionary value) => jsonStream.Length, + expected); + } + } + + private static void JsonReadBenchmark(string dataSetName, int innerLoopIterations, List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Encoding utf8Encoding = new UTF8Encoding(); + JsonSerializer jsonSerializer = JsonSerializer.Create(SchematizedMicroBenchmarkSuite.JsonSettings); + using (MemoryStream jsonStream = new MemoryStream(BenchmarkSuiteBase.InitialCapacity)) + using (StreamWriter textWriter = new StreamWriter(jsonStream, utf8Encoding)) + using (JsonTextWriter jsonWriter = new JsonTextWriter(textWriter)) + { + jsonSerializer.Converters.Add(new Utf8StringJsonConverter()); + + foreach (Dictionary tableValue in expected) + { + jsonSerializer.Serialize(jsonWriter, tableValue); + jsonWriter.Flush(); + expectedSerialized.Add(jsonStream.ToArray()); + jsonStream.SetLength(0); + } + } + + BenchmarkContext ignoredContext = default; + jsonSerializer.Converters.Add(new Utf8StringJsonConverter()); + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Read", + dataSetName, + "JSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, byte[] tableValue) => + { + using (MemoryStream jsonStream = new MemoryStream(tableValue)) + using (StreamReader textReader = new StreamReader(jsonStream, utf8Encoding)) + using (JsonTextReader jsonReader = new JsonTextReader(textReader)) + { + while (jsonReader.Read()) + { + // Just visit the entire structure without materializing any of the values. + } + } + }, + (ref BenchmarkContext _, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + + private static void BsonWriteBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + using (BsonRowGenerator writer = new BsonRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver)) + { + BenchmarkContext ignoredContext = default; + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Write", + dataSetName, + "BSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, Dictionary tableValue) => + { + writer.Reset(); + writer.WriteBuffer(tableValue); + }, + (ref BenchmarkContext _, Dictionary tableValue) => writer.Length, + expected); + } + } + + private static void BsonReadBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + using (BsonRowGenerator writer = new BsonRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver)) + { + foreach (Dictionary tableValue in expected) + { + writer.Reset(); + writer.WriteBuffer(tableValue); + expectedSerialized.Add(writer.ToArray()); + } + } + + BenchmarkContext ignoredContext = default; + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Read", + dataSetName, + "BSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, byte[] tableValue) => + { + using (MemoryStream stm = new MemoryStream(tableValue)) + using (BsonBinaryReader bsonReader = new BsonBinaryReader(stm)) + { + bsonReader.VisitBsonDocument(); + } + }, + (ref BenchmarkContext _, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + + private static void LayoutWriteBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + PatchWriter = new WriteRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Write", + dataSetName, + "Layout", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, Dictionary dict) => + { + ctx.PatchWriter.Reset(); + Result r = ctx.PatchWriter.DispatchLayout(layout, dict); + ResultAssert.IsSuccess(r); + }, + (ref BenchmarkContext ctx, Dictionary _) => ctx.PatchWriter.Length, + expected); + } + + private static void LayoutReadBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + StreamingWriter = new StreamingRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + foreach (Dictionary tableValue in expected) + { + context.StreamingWriter.Reset(); + + Result r = context.StreamingWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + expectedSerialized.Add(context.StreamingWriter.ToArray()); + } + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Read", + dataSetName, + "Layout", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, byte[] tableValue) => + { + VisitRowGenerator visitor = new VisitRowGenerator(tableValue.AsSpan(), resolver); + Result r = visitor.DispatchLayout(layout); + ResultAssert.IsSuccess(r); + }, + (ref BenchmarkContext ctx, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + + private static void StreamingWriteBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + StreamingWriter = new StreamingRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Write", + dataSetName, + "HybridRow", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, Dictionary tableValue) => + { + ctx.StreamingWriter.Reset(); + + Result r = ctx.StreamingWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + }, + (ref BenchmarkContext ctx, Dictionary _) => ctx.StreamingWriter.Length, + expected); + } + + private static void StreamingReadBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + StreamingWriter = new StreamingRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + foreach (Dictionary tableValue in expected) + { + context.StreamingWriter.Reset(); + + Result r = context.StreamingWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + expectedSerialized.Add(context.StreamingWriter.ToArray()); + } + + MicroBenchmarkSuiteBase.Benchmark( + "Schematized", + "Read", + dataSetName, + "HybridRow", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, byte[] tableValue) => + { + RowBuffer row = new RowBuffer(tableValue.AsSpan(), HybridRowVersion.V1, resolver); + RowReader reader = new RowReader(ref row); + reader.VisitReader(); + }, + (ref BenchmarkContext ctx, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData.cs b/dotnet/src/HybridRow.Tests.Perf/TestData.cs new file mode 100644 index 0000000..9e65909 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/TestData.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + /// + /// Names of assets in the TestData folder. + /// + internal class TestData + { + /// + /// The folder to which TestData assets should be copied during deployment. + /// + public const string Target = "TestData"; + + public const string SchemaFile = @"TestData\CassandraHotelSchema.json"; + public const string HotelExpected = @"TestData\HotelSchemaExpected.hr"; + public const string RoomsExpected = @"TestData\RoomsSchemaExpected.hr"; + public const string GuestsExpected = @"TestData\GuestsSchemaExpected.hr"; + public const string Messages1KExpected = @"TestData\Messages1KExpected.hr"; + } +} diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/CassandraHotelSchema.json b/dotnet/src/HybridRow.Tests.Perf/TestData/CassandraHotelSchema.json new file mode 100644 index 0000000..c27a3bc --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/TestData/CassandraHotelSchema.json @@ -0,0 +1,76 @@ +// Partial implementation of Cassandra Hotel Schema described here:: +// https://www.oreilly.com/ideas/cassandra-data-modeling +{ + "name": "Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel", + "schemas": [ + { + "name": "PostalCode", + "id": 1, + "type": "schema", + "properties": [ + { "path": "zip", "type": { "type": "int32", "storage": "fixed" } }, + { "path": "plus4", "type": { "type": "int16", "storage": "sparse" } } + ] + }, + { + "name": "Address", + "id": 2, + "type": "schema", + "properties": [ + { "path": "street", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "city", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "state", "type": { "type": "utf8", "storage": "fixed", "length": 2 } }, + { "path": "postal_code", "type": { "type": "schema", "name": "PostalCode" } } + ] + }, + { + "name": "Hotels", + "id": 3, + "type": "schema", + "partitionkeys": [{ "path": "hotel_id" }], + "properties": [ + { "path": "hotel_id", "type": { "type": "utf8", "storage": "fixed", "length": 8 } }, + { "path": "name", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "phone", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "address", "type": { "type": "schema", "name": "Address", "immutable": true } } + ] + }, + { + "name": "Available_Rooms_By_Hotel_Date", + "id": 4, + "type": "schema", + "partitionkeys": [{ "path": "hotel_id" }], + "primarykeys": [{ "path": "date" }, { "path": "room_number", "direction": "desc" }], + "properties": [ + { "path": "hotel_id", "type": { "type": "utf8", "storage": "fixed", "length": 8 } }, + { "path": "date", "type": { "type": "datetime", "storage": "fixed" } }, + { "path": "room_number", "type": { "type": "uint8", "storage": "fixed" } }, + { "path": "is_available", "type": { "type": "bool", "storage": "fixed" } } + ] + }, + { + "name": "Guests", + "id": 5, + "type": "schema", + "partitionkeys": [{ "path": "guest_id" }], + "primarykeys": [{"path": "first_name"}, {"path": "phone_numbers", "direction": "desc"}], + "properties": [ + { "path": "guest_id", "type": { "type": "guid", "storage": "fixed" } }, + { "path": "first_name", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "last_name", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "title", "type": { "type": "utf8", "storage": "variable", "length": 20 } }, + { "path": "emails", "type": { "type": "array", "items": { "type": "utf8", "nullable": false } } }, + { "path": "phone_numbers", "type": { "type": "array", "items": { "type": "utf8", "nullable": false } } }, + { + "path": "addresses", + "type": { + "type": "map", + "keys": { "type": "utf8", "nullable": false }, + "values": { "type": "schema", "name": "Address", "immutable": true, "nullable": false } + } + }, + { "path": "confirm_number", "type": { "type": "utf8", "storage": "variable" } } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/CassandraHotelSchema.proto b/dotnet/src/HybridRow.Tests.Perf/TestData/CassandraHotelSchema.proto new file mode 100644 index 0000000..abb1b3e --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/TestData/CassandraHotelSchema.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; +package Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel; + +import "google/protobuf/wrappers.proto"; + +option csharp_namespace = "Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf.CassandraHotel.Protobuf"; + +message PostalCode { + google.protobuf.Int32Value zip = 1; + google.protobuf.Int32Value plus4 = 2; +} + +message Address { + google.protobuf.StringValue street = 1; + google.protobuf.StringValue city = 2; + google.protobuf.StringValue state = 3; + PostalCode postal_code = 4; +} + +message Hotels { + google.protobuf.StringValue hotel_id = 1; + google.protobuf.StringValue name = 2; + google.protobuf.StringValue phone = 3; + Address address = 4; +} + +message Available_Rooms_By_Hotel_Date { + google.protobuf.StringValue hotel_id = 1; + google.protobuf.Int64Value date = 2; // datetime + google.protobuf.Int32Value room_number = 3; + google.protobuf.BoolValue is_available = 4; +} + +message Guests { + google.protobuf.StringValue guest_id = 1; // guid + google.protobuf.StringValue first_name = 2; + google.protobuf.StringValue last_name = 3; + google.protobuf.StringValue title = 4; + repeated string emails = 5; + repeated string phone_numbers = 6; + map addresses = 7; + google.protobuf.StringValue confirm_number = 8; +} diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/CombinedScriptsData.hr b/dotnet/src/HybridRow.Tests.Perf/TestData/CombinedScriptsData.hr new file mode 100644 index 0000000..cf62dc7 Binary files /dev/null and b/dotnet/src/HybridRow.Tests.Perf/TestData/CombinedScriptsData.hr differ diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/GuestsSchemaExpected.hr b/dotnet/src/HybridRow.Tests.Perf/TestData/GuestsSchemaExpected.hr new file mode 100644 index 0000000..a275dea Binary files /dev/null and b/dotnet/src/HybridRow.Tests.Perf/TestData/GuestsSchemaExpected.hr differ diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/HotelSchemaExpected.hr b/dotnet/src/HybridRow.Tests.Perf/TestData/HotelSchemaExpected.hr new file mode 100644 index 0000000..2635ecd Binary files /dev/null and b/dotnet/src/HybridRow.Tests.Perf/TestData/HotelSchemaExpected.hr differ diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/Messages1KExpected.hr b/dotnet/src/HybridRow.Tests.Perf/TestData/Messages1KExpected.hr new file mode 100644 index 0000000..d37d502 Binary files /dev/null and b/dotnet/src/HybridRow.Tests.Perf/TestData/Messages1KExpected.hr differ diff --git a/dotnet/src/HybridRow.Tests.Perf/TestData/RoomsSchemaExpected.hr b/dotnet/src/HybridRow.Tests.Perf/TestData/RoomsSchemaExpected.hr new file mode 100644 index 0000000..2ecf0d3 Binary files /dev/null and b/dotnet/src/HybridRow.Tests.Perf/TestData/RoomsSchemaExpected.hr differ diff --git a/dotnet/src/HybridRow.Tests.Perf/UnschematizedMicroBenchmarkSuite.cs b/dotnet/src/HybridRow.Tests.Perf/UnschematizedMicroBenchmarkSuite.cs new file mode 100644 index 0000000..0bb0b01 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Perf/UnschematizedMicroBenchmarkSuite.cs @@ -0,0 +1,338 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Internal; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using MongoDB.Bson.IO; + using Newtonsoft.Json; + + /// Tests involving fully (or mostly) unschematized test data. + [TestClass] + public sealed class UnschematizedMicroBenchmarkSuite : MicroBenchmarkSuiteBase + { + private static readonly JsonSerializerSettings JsonSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented + }; + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.Messages1KExpected, TestData.Target)] + public async Task Messages1KWriteBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 10; +#endif + string expectedFile = TestData.Messages1KExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + UnschematizedMicroBenchmarkSuite.JsonModelWriteBenchmark( + resolver, + "TypedJsonHybridRowSchema", + "Messages1K", + innerLoopIterations, + expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.Messages1KExpected, TestData.Target)] + public async Task Messages1KReadBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 10; +#endif + string expectedFile = TestData.Messages1KExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + UnschematizedMicroBenchmarkSuite.JsonModelReadBenchmark( + resolver, + "TypedJsonHybridRowSchema", + "Messages1K", + innerLoopIterations, + expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.Messages1KExpected, TestData.Target)] + public async Task BsonMessages1KWriteBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 10; +#endif + string expectedFile = TestData.Messages1KExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + UnschematizedMicroBenchmarkSuite.BsonWriteBenchmark("Messages1K", innerLoopIterations, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.Messages1KExpected, TestData.Target)] + public async Task BsonMessages1KReadBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 10; +#endif + string expectedFile = TestData.Messages1KExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + UnschematizedMicroBenchmarkSuite.BsonReadBenchmark("Messages1K", innerLoopIterations, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.Messages1KExpected, TestData.Target)] + public async Task JsonMessages1KWriteBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 10; +#endif + string expectedFile = TestData.Messages1KExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + UnschematizedMicroBenchmarkSuite.JsonWriteBenchmark("Messages1K", innerLoopIterations, expected); + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(TestData.Messages1KExpected, TestData.Target)] + public async Task JsonMessages1KReadBenchmarkAsync() + { +#if DEBUG + const int innerLoopIterations = 1; +#else + const int innerLoopIterations = 10; +#endif + string expectedFile = TestData.Messages1KExpected; + (List> expected, LayoutResolverNamespace resolver) = await this.LoadExpectedAsync(expectedFile); + UnschematizedMicroBenchmarkSuite.JsonReadBenchmark("Messages1K", innerLoopIterations, expected); + } + + private static void JsonModelWriteBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + JsonModelWriter = new JsonModelRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + MicroBenchmarkSuiteBase.Benchmark( + "Unschematized", + "Write", + dataSetName, + "HybridRowSparse", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, Dictionary tableValue) => + { + ctx.JsonModelWriter.Reset(); + + Result r = ctx.JsonModelWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + }, + (ref BenchmarkContext ctx, Dictionary tableValue) => ctx.JsonModelWriter.Length, + expected); + } + + private static void JsonModelReadBenchmark( + LayoutResolverNamespace resolver, + string schemaName, + string dataSetName, + int innerLoopIterations, + List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Layout layout = resolver.Resolve(resolver.Namespace.Schemas.Find(x => x.Name == schemaName).SchemaId); + BenchmarkContext context = new BenchmarkContext + { + JsonModelWriter = new JsonModelRowGenerator(BenchmarkSuiteBase.InitialCapacity, layout, resolver) + }; + + foreach (Dictionary tableValue in expected) + { + context.JsonModelWriter.Reset(); + + Result r = context.JsonModelWriter.WriteBuffer(tableValue); + ResultAssert.IsSuccess(r); + expectedSerialized.Add(context.JsonModelWriter.ToArray()); + } + + MicroBenchmarkSuiteBase.Benchmark( + "Unschematized", + "Read", + dataSetName, + "HybridRowSparse", + innerLoopIterations, + ref context, + (ref BenchmarkContext ctx, byte[] tableValue) => + { + RowBuffer row = new RowBuffer(tableValue.AsSpan(), HybridRowVersion.V1, resolver); + RowReader reader = new RowReader(ref row); + reader.VisitReader(); + }, + (ref BenchmarkContext ctx, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + + private static void JsonWriteBenchmark( + string dataSetName, + int innerLoopIterations, + List> expected) + { + Encoding utf8Encoding = new UTF8Encoding(); + JsonSerializer jsonSerializer = JsonSerializer.Create(UnschematizedMicroBenchmarkSuite.JsonSettings); + using (MemoryStream jsonStream = new MemoryStream(BenchmarkSuiteBase.InitialCapacity)) + using (StreamWriter textWriter = new StreamWriter(jsonStream, utf8Encoding)) + using (JsonTextWriter jsonWriter = new JsonTextWriter(textWriter)) + { + BenchmarkContext ignoredContext = default; + jsonSerializer.Converters.Add(new Utf8StringJsonConverter()); + + MicroBenchmarkSuiteBase.Benchmark( + "Unschematized", + "Write", + dataSetName, + "JSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, Dictionary tableValue) => + { + jsonStream.SetLength(0); + jsonSerializer.Serialize(jsonWriter, tableValue); + jsonWriter.Flush(); + }, + (ref BenchmarkContext _, Dictionary value) => jsonStream.Length, + expected); + } + } + + private static void JsonReadBenchmark(string dataSetName, int innerLoopIterations, List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + Encoding utf8Encoding = new UTF8Encoding(); + JsonSerializer jsonSerializer = JsonSerializer.Create(UnschematizedMicroBenchmarkSuite.JsonSettings); + using (MemoryStream jsonStream = new MemoryStream(BenchmarkSuiteBase.InitialCapacity)) + using (StreamWriter textWriter = new StreamWriter(jsonStream, utf8Encoding)) + using (JsonTextWriter jsonWriter = new JsonTextWriter(textWriter)) + { + jsonSerializer.Converters.Add(new Utf8StringJsonConverter()); + + foreach (Dictionary tableValue in expected) + { + jsonSerializer.Serialize(jsonWriter, tableValue); + jsonWriter.Flush(); + expectedSerialized.Add(jsonStream.ToArray()); + jsonStream.SetLength(0); + } + } + + BenchmarkContext ignoredContext = default; + jsonSerializer.Converters.Add(new Utf8StringJsonConverter()); + + MicroBenchmarkSuiteBase.Benchmark( + "Unschematized", + "Read", + dataSetName, + "JSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, byte[] tableValue) => + { + using (MemoryStream jsonStream = new MemoryStream(tableValue)) + using (StreamReader textReader = new StreamReader(jsonStream, utf8Encoding)) + using (JsonTextReader jsonReader = new JsonTextReader(textReader)) + { + while (jsonReader.Read()) + { + // Just visit the entire structure without materializing any of the values. + } + } + }, + (ref BenchmarkContext _, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + + private static void BsonWriteBenchmark(string dataSetName, int innerLoopIterations, List> expected) + { + using (BsonJsonModelRowGenerator writer = new BsonJsonModelRowGenerator(BenchmarkSuiteBase.InitialCapacity)) + { + BenchmarkContext ignoredContext = default; + + MicroBenchmarkSuiteBase.Benchmark( + "Unschematized", + "Write", + dataSetName, + "BSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, Dictionary tableValue) => + { + writer.Reset(); + writer.WriteBuffer(tableValue); + }, + (ref BenchmarkContext _, Dictionary tableValue) => writer.Length, + expected); + } + } + + private static void BsonReadBenchmark(string dataSetName, int innerLoopIterations, List> expected) + { + // Serialize input data to sequence of byte buffers. + List expectedSerialized = new List(expected.Count); + using (BsonJsonModelRowGenerator writer = new BsonJsonModelRowGenerator(BenchmarkSuiteBase.InitialCapacity)) + { + foreach (Dictionary tableValue in expected) + { + writer.Reset(); + writer.WriteBuffer(tableValue); + expectedSerialized.Add(writer.ToArray()); + } + } + + BenchmarkContext ignoredContext = default; + MicroBenchmarkSuiteBase.Benchmark( + "Unschematized", + "Read", + dataSetName, + "BSON", + innerLoopIterations, + ref ignoredContext, + (ref BenchmarkContext _, byte[] tableValue) => + { + using (MemoryStream stm = new MemoryStream(tableValue)) + using (BsonBinaryReader bsonReader = new BsonBinaryReader(stm)) + { + bsonReader.VisitBsonDocument(); + } + }, + (ref BenchmarkContext _, byte[] tableValue) => tableValue.Length, + expectedSerialized); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/App.config b/dotnet/src/HybridRow.Tests.Unit/App.config new file mode 100644 index 0000000..02249c3 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/App.config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/dotnet/src/HybridRow.Tests.Unit/ArrayAssert.cs b/dotnet/src/HybridRow.Tests.Unit/ArrayAssert.cs new file mode 100644 index 0000000..27edd37 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/ArrayAssert.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + + internal static class ArrayAssert + { + public static void AreEqual(T[] expected, T[] actual) + { + if (expected == null) + { + Assert.IsNull(actual); + return; + } + + Assert.IsNotNull(actual); + Assert.AreEqual(expected.Length, actual.Length); + for (int i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], actual[i]); + } + } + + public static void AreEqual(T[] expected, T[] actual, string message) + { + if (expected == null) + { + Assert.IsNull(actual, message); + return; + } + + Assert.IsNotNull(actual, message); + Assert.AreEqual(expected.Length, actual.Length, message); + for (int i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], actual[i], message); + } + } + + public static void AreEqual(T[] expected, T[] actual, string message, params object[] parameters) + { + if (expected == null) + { + Assert.IsNull(actual, message, parameters); + return; + } + + Assert.IsNotNull(actual, message, parameters); + Assert.AreEqual(expected.Length, actual.Length, message, parameters); + for (int i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], actual[i], message, parameters); + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/AssertThrowsException.cs b/dotnet/src/HybridRow.Tests.Unit/AssertThrowsException.cs new file mode 100644 index 0000000..5905946 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/AssertThrowsException.cs @@ -0,0 +1,137 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + internal static class AssertThrowsException + { + /// + /// Tests whether the code specified by delegate throws exact given + /// exception of type (and not of derived type) and throws + /// AssertFailedException + /// if code does not throws exception or throws exception of type other than + /// . + /// + /// Delegate to code to be tested and which is expected to throw exception. + /// Type of exception expected to be thrown. + /// + /// Thrown if + /// does not throws exception of type . + /// + /// The exception that was thrown. + public static T ThrowsException(Action action) + where T : Exception + { + return AssertThrowsException.ThrowsException(action, string.Empty, null); + } + + /// + /// Tests whether the code specified by delegate throws exact given + /// exception of type (and not of derived type) and throws + /// AssertFailedException + /// if code does not throws exception or throws exception of type other than + /// . + /// + /// Delegate to code to be tested and which is expected to throw exception. + /// + /// The message to include in the exception when does + /// not throws exception of type . + /// + /// Type of exception expected to be thrown. + /// + /// Thrown if + /// does not throws exception of type . + /// + /// The exception that was thrown. + public static T ThrowsException(Action action, string message) + where T : Exception + { + return AssertThrowsException.ThrowsException(action, message, null); + } + + /// + /// Tests whether the code specified by delegate throws exact given + /// exception of type (and not of derived type) and throws + /// AssertFailedException + /// if code does not throws exception or throws exception of type other than + /// . + /// + /// Delegate to code to be tested and which is expected to throw exception. + /// + /// The message to include in the exception when does + /// not throws exception of type . + /// + /// An array of parameters to use when formatting . + /// Type of exception expected to be thrown. + /// + /// Thrown if + /// does not throws exception of type . + /// + /// The exception that was thrown. + public static T ThrowsException(Action action, string message, params object[] parameters) + where T : Exception + { + if (action == null) + { + throw new ArgumentNullException(nameof(action)); + } + + if (message == null) + { + throw new ArgumentNullException(nameof(message)); + } + + try + { + action(); + } + catch (Exception ex) + { + if (typeof(T) != ex.GetType()) + { + Assert.Fail( + string.Format( + "Threw exception {2}, but exception {1} was expected. {0}\nException Message: {3}\nStack Trace: {4}", + (object)AssertThrowsException.ReplaceNulls(message), + (object)typeof(T).Name, + (object)ex.GetType().Name, + (object)ex.Message, + (object)ex.StackTrace), + parameters); + } + + return (T)ex; + } + + Assert.Fail( + string.Format( + "No exception thrown. {1} exception was expected. {0}", + AssertThrowsException.ReplaceNulls(message), + typeof(T).Name), + parameters); + + return default; + } + + /// + /// Safely converts an object to a string, handling null values and null characters. Null + /// values are converted to "(null)". Null characters are converted to "\\0". + /// + /// The object to convert to a string. + /// The converted string. + internal static string ReplaceNulls(object input) + { + string input1 = input?.ToString(); + if (input1 == null) + { + return "(null)"; + } + + return Assert.ReplaceNullChars(input1); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CrossVersioningUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/CrossVersioningUnitTests.cs new file mode 100644 index 0000000..8375643 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CrossVersioningUnitTests.cs @@ -0,0 +1,494 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Public Fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + [TestClass] + [SuppressMessage("Naming", "DontUseVarForVariableTypes", Justification = "The types here are anonymous.")] + [DeploymentItem(CrossVersioningUnitTests.SchemaFile, "TestData")] + [DeploymentItem(CrossVersioningUnitTests.ExpectedFile, "TestData")] + public sealed class CrossVersioningUnitTests + { + private const string SchemaFile = @"TestData\CrossVersioningSchema.json"; + private const string ExpectedFile = @"TestData\CrossVersioningExpected.json"; + private static readonly DateTime SampleDateTime = DateTime.Parse("2018-08-14 02:05:00.0000000"); + private static readonly Guid SampleGuid = Guid.Parse("{2A9C25B9-922E-4611-BB0A-244A9496503C}"); + private static readonly Float128 SampleFloat128 = new Float128(0, 42); + private static readonly UnixDateTime SampleUnixDateTime = new UnixDateTime(42); + private static readonly MongoDbObjectId SampleMongoDbObjectId = new MongoDbObjectId(704643072U, 0); // 42 in big-endian + + private Namespace schema; + private LayoutResolver resolver; + private Expected expected; + + [TestInitialize] + public void ParseNamespace() + { + string json = File.ReadAllText(CrossVersioningUnitTests.SchemaFile); + this.schema = Namespace.Parse(json); + json = File.ReadAllText(CrossVersioningUnitTests.ExpectedFile); + this.expected = JsonConvert.DeserializeObject(json); + this.resolver = new LayoutResolverNamespace(this.schema); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionWriteFixed() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Fixed").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool", value: true); + d.LayoutCodeSwitch("int8", value: (sbyte)-86); + d.LayoutCodeSwitch("int16", value: (short)-21846); + d.LayoutCodeSwitch("int32", value: -1431655766); + d.LayoutCodeSwitch("int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("float128", value: CrossVersioningUnitTests.SampleFloat128); + d.LayoutCodeSwitch("decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("datetime", value: CrossVersioningUnitTests.SampleDateTime); + d.LayoutCodeSwitch("unixdatetime", value: CrossVersioningUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("guid", value: CrossVersioningUnitTests.SampleGuid); + d.LayoutCodeSwitch("mongodbobjectid", value: CrossVersioningUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + + Assert.AreEqual(this.expected.CrossVersionFixed, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionReadFixed() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Fixed").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.ReadFrom(this.resolver, this.expected.CrossVersionFixed); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool", value: true); + d.LayoutCodeSwitch("int8", value: (sbyte)-86); + d.LayoutCodeSwitch("int16", value: (short)-21846); + d.LayoutCodeSwitch("int32", value: -1431655766); + d.LayoutCodeSwitch("int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("float128", value: CrossVersioningUnitTests.SampleFloat128); + d.LayoutCodeSwitch("decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("datetime", value: CrossVersioningUnitTests.SampleDateTime); + d.LayoutCodeSwitch("unixdatetime", value: CrossVersioningUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("guid", value: CrossVersioningUnitTests.SampleGuid); + d.LayoutCodeSwitch("mongodbobjectid", value: CrossVersioningUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionWriteNullFixed() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Fixed").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + Assert.AreEqual(this.expected.CrossVersionNullFixed, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionReadNullFixed() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Fixed").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool"); + d.LayoutCodeSwitch("int8"); + d.LayoutCodeSwitch("int16"); + d.LayoutCodeSwitch("int32"); + d.LayoutCodeSwitch("int64"); + d.LayoutCodeSwitch("uint8"); + d.LayoutCodeSwitch("uint16"); + d.LayoutCodeSwitch("uint32"); + d.LayoutCodeSwitch("uint64"); + d.LayoutCodeSwitch("float32"); + d.LayoutCodeSwitch("float64"); + d.LayoutCodeSwitch("float128"); + d.LayoutCodeSwitch("decimal"); + d.LayoutCodeSwitch("datetime"); + d.LayoutCodeSwitch("unixdatetime"); + d.LayoutCodeSwitch("guid"); + d.LayoutCodeSwitch("mongodbobjectid"); + d.LayoutCodeSwitch("utf8"); + d.LayoutCodeSwitch("binary"); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionWriteVariable() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Variable").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + d.LayoutCodeSwitch("varint", value: -6148914691236517206L); + d.LayoutCodeSwitch("varuint", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + + Assert.AreEqual(this.expected.CrossVersionVariable, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionReadVariable() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Variable").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.ReadFrom(this.resolver, this.expected.CrossVersionVariable); + d.LayoutCodeSwitch("varint", value: -6148914691236517206L); + d.LayoutCodeSwitch("varuint", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionWriteNullVariable() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Variable").SchemaId); + Assert.IsNotNull(layout); + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + Assert.AreEqual(this.expected.CrossVersionNullVariable, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionReadNullVariable() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Variable").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + d.LayoutCodeSwitch("varint"); + d.LayoutCodeSwitch("varuint"); + d.LayoutCodeSwitch("utf8"); + d.LayoutCodeSwitch("binary"); + } + + [TestMethod] + [Owner("jthunter")] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1118:ParameterMustNotSpanMultipleLines", Justification = "Test code.")] + public void CrossVersionWriteSparse() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Sparse").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool", value: true); + d.LayoutCodeSwitch("int8", value: (sbyte)-86); + d.LayoutCodeSwitch("int16", value: (short)-21846); + d.LayoutCodeSwitch("int32", value: -1431655766); + d.LayoutCodeSwitch("int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("float128", value: CrossVersioningUnitTests.SampleFloat128); + d.LayoutCodeSwitch("decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("datetime", value: CrossVersioningUnitTests.SampleDateTime); + d.LayoutCodeSwitch("unixdatetime", value: CrossVersioningUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("guid", value: CrossVersioningUnitTests.SampleGuid); + d.LayoutCodeSwitch("mongodbobjectid", value: CrossVersioningUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("array_t", value: new sbyte[] { -86, -86, -86 }); + d.LayoutCodeSwitch("array_t>", value: new[] { new float[] { 1, 2, 3 }, new float[] { 1, 2, 3 } }); + d.LayoutCodeSwitch("array_t", value: new[] { "abc", "def", "hij" }); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(-6148914691236517206L, -6148914691236517206L)); + d.LayoutCodeSwitch("tuple>", value: Tuple.Create(NullValue.Default, Tuple.Create((sbyte)-86, (sbyte)-86))); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(false, new Point(1, 2))); + d.LayoutCodeSwitch("set_t", value: new[] { "abc", "efg", "xzy" }); + d.LayoutCodeSwitch("set_t>", value: new[] { new sbyte[] { 1, 2, 3 }, new sbyte[] { 4, 5, 6 }, new sbyte[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t>", value: new[] { new[] { 1, 2, 3 }, new[] { 4, 5, 6 }, new[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t", value: new[] { new Point(1, 2), new Point(3, 4), new Point(5, 6) }); + d.LayoutCodeSwitch("map_t", value: new[] { Tuple.Create("Mark", "Luke"), Tuple.Create("Harrison", "Han") }); + d.LayoutCodeSwitch( + "map_t>", + value: new[] { Tuple.Create((sbyte)1, new sbyte[] { 1, 2, 3 }), Tuple.Create((sbyte)2, new sbyte[] { 4, 5, 6 }) }); + + d.LayoutCodeSwitch( + "map_t>", + value: new[] + { + Tuple.Create((short)1, new[] { Tuple.Create(1, 2), Tuple.Create(3, 4) }), + Tuple.Create((short)2, new[] { Tuple.Create(5, 6), Tuple.Create(7, 8) }), + }); + + d.LayoutCodeSwitch( + "map_t", + value: new[] + { + Tuple.Create(1.0, new Point(1, 2)), + Tuple.Create(2.0, new Point(3, 4)), + Tuple.Create(3.0, new Point(5, 6)), + }); + + Assert.AreEqual(this.expected.CrossVersionSparse, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1118:ParameterMustNotSpanMultipleLines", Justification = "Test code.")] + public void CrossVersionReadSparse() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Sparse").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.ReadFrom(this.resolver, this.expected.CrossVersionSparse); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool", value: true); + d.LayoutCodeSwitch("int8", value: (sbyte)-86); + d.LayoutCodeSwitch("int16", value: (short)-21846); + d.LayoutCodeSwitch("int32", value: -1431655766); + d.LayoutCodeSwitch("int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("float128", value: CrossVersioningUnitTests.SampleFloat128); + d.LayoutCodeSwitch("decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("datetime", value: CrossVersioningUnitTests.SampleDateTime); + d.LayoutCodeSwitch("unixdatetime", value: CrossVersioningUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("guid", value: CrossVersioningUnitTests.SampleGuid); + d.LayoutCodeSwitch("mongodbobjectid", value: CrossVersioningUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("array_t", value: new sbyte[] { -86, -86, -86 }); + d.LayoutCodeSwitch("array_t>", value: new[] { new float[] { 1, 2, 3 }, new float[] { 1, 2, 3 } }); + d.LayoutCodeSwitch("array_t", value: new[] { "abc", "def", "hij" }); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(-6148914691236517206L, -6148914691236517206L)); + d.LayoutCodeSwitch("tuple>", value: Tuple.Create(NullValue.Default, Tuple.Create((sbyte)-86, (sbyte)-86))); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(false, new Point(1, 2))); + d.LayoutCodeSwitch("set_t", value: new[] { "abc", "efg", "xzy" }); + d.LayoutCodeSwitch("set_t>", value: new[] { new sbyte[] { 1, 2, 3 }, new sbyte[] { 4, 5, 6 }, new sbyte[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t>", value: new[] { new[] { 1, 2, 3 }, new[] { 4, 5, 6 }, new[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t", value: new[] { new Point(1, 2), new Point(3, 4), new Point(5, 6) }); + d.LayoutCodeSwitch("map_t", value: new[] { Tuple.Create("Mark", "Luke"), Tuple.Create("Harrison", "Han") }); + d.LayoutCodeSwitch( + "map_t>", + value: new[] { Tuple.Create((sbyte)1, new sbyte[] { 1, 2, 3 }), Tuple.Create((sbyte)2, new sbyte[] { 4, 5, 6 }) }); + + d.LayoutCodeSwitch( + "map_t>", + value: new[] + { + Tuple.Create((short)1, new[] { Tuple.Create(1, 2), Tuple.Create(3, 4) }), + Tuple.Create((short)2, new[] { Tuple.Create(5, 6), Tuple.Create(7, 8) }), + }); + + d.LayoutCodeSwitch( + "map_t", + value: new[] + { + Tuple.Create(2.0, new Point(3, 4)), + Tuple.Create(3.0, new Point(5, 6)), + Tuple.Create(1.0, new Point(1, 2)), + }); + } + + [TestMethod] + [Owner("jthunter")] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1118:ParameterMustNotSpanMultipleLines", Justification = "Test code.")] + public void CrossVersionDeleteSparse() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Sparse").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.ReadFrom(this.resolver, this.expected.CrossVersionSparse); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool", value: true); + d.LayoutCodeSwitch("int8", value: (sbyte)-86); + d.LayoutCodeSwitch("int16", value: (short)-21846); + d.LayoutCodeSwitch("int32", value: -1431655766); + d.LayoutCodeSwitch("int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("float128", value: CrossVersioningUnitTests.SampleFloat128); + d.LayoutCodeSwitch("decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("datetime", value: CrossVersioningUnitTests.SampleDateTime); + d.LayoutCodeSwitch("unixdatetime", value: CrossVersioningUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("guid", value: CrossVersioningUnitTests.SampleGuid); + d.LayoutCodeSwitch("mongodbobjectid", value: CrossVersioningUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("array_t", value: new sbyte[] { -86, -86, -86 }); + d.LayoutCodeSwitch("array_t>", value: new[] { new float[] { 1, 2, 3 }, new float[] { 1, 2, 3 } }); + d.LayoutCodeSwitch("array_t", value: new[] { "abc", "def", "hij" }); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(-6148914691236517206L, -6148914691236517206L)); + d.LayoutCodeSwitch("tuple>", value: Tuple.Create(NullValue.Default, Tuple.Create((sbyte)-86, (sbyte)-86))); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(false, new Point(1, 2))); + d.LayoutCodeSwitch("set_t", value: new[] { "abc", "efg", "xzy" }); + d.LayoutCodeSwitch("set_t>", value: new[] { new sbyte[] { 1, 2, 3 }, new sbyte[] { 4, 5, 6 }, new sbyte[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t>", value: new[] { new[] { 1, 2, 3 }, new[] { 4, 5, 6 }, new[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t", value: new[] { new Point(1, 2), new Point(3, 4), new Point(5, 6) }); + d.LayoutCodeSwitch("map_t", value: new[] { Tuple.Create("Mark", "Luke"), Tuple.Create("Harrison", "Han") }); + d.LayoutCodeSwitch( + "map_t>", + value: new[] { Tuple.Create((sbyte)1, new sbyte[] { 1, 2, 3 }), Tuple.Create((sbyte)2, new sbyte[] { 4, 5, 6 }) }); + + d.LayoutCodeSwitch( + "map_t>", + value: new[] + { + Tuple.Create((short)1, new[] { Tuple.Create(1, 2), Tuple.Create(3, 4) }), + Tuple.Create((short)2, new[] { Tuple.Create(5, 6), Tuple.Create(7, 8) }), + }); + + d.LayoutCodeSwitch( + "map_t", + value: new[] + { + Tuple.Create(1.0, new Point(1, 2)), + Tuple.Create(2.0, new Point(3, 4)), + Tuple.Create(3.0, new Point(5, 6)), + }); + + Assert.AreEqual(this.expected.CrossVersionNullSparse, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionWriteNullSparse() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Sparse").SchemaId); + Assert.IsNotNull(layout); + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + Assert.AreEqual(this.expected.CrossVersionNullSparse, d.RowToHex()); + } + + [TestMethod] + [Owner("jthunter")] + public void CrossVersionReadNullSparse() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Sparse").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.ReadFrom(this.resolver, this.expected.CrossVersionNullSparse); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool"); + d.LayoutCodeSwitch("int8"); + d.LayoutCodeSwitch("int16"); + d.LayoutCodeSwitch("int32"); + d.LayoutCodeSwitch("int64"); + d.LayoutCodeSwitch("uint8"); + d.LayoutCodeSwitch("uint16"); + d.LayoutCodeSwitch("uint32"); + d.LayoutCodeSwitch("uint64"); + d.LayoutCodeSwitch("float32"); + d.LayoutCodeSwitch("float64"); + d.LayoutCodeSwitch("float128"); + d.LayoutCodeSwitch("decimal"); + d.LayoutCodeSwitch("datetime"); + d.LayoutCodeSwitch("unixdatetime"); + d.LayoutCodeSwitch("guid"); + d.LayoutCodeSwitch("mongodbobjectid"); + d.LayoutCodeSwitch("utf8"); + d.LayoutCodeSwitch("binary"); + } + + private sealed class Point : IDispatchable + { + public readonly int X; + public readonly int Y; + + public Point(int x, int y) + { + this.X = x; + this.Y = y; + } + + public override bool Equals(object obj) + { + if (obj is null) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Point point && this.Equals(point); + } + + public override int GetHashCode() + { + unchecked + { + return (this.X.GetHashCode() * 397) ^ this.Y.GetHashCode(); + } + } + + void IDispatchable.Dispatch(ref RowOperationDispatcher dispatcher, ref RowCursor scope) + { + dispatcher.LayoutCodeSwitch(ref scope, "x", value: this.X); + dispatcher.LayoutCodeSwitch(ref scope, "y", value: this.Y); + } + + private bool Equals(Point other) + { + return this.X == other.X && this.Y == other.Y; + } + } + + [SuppressMessage("Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses", Justification = "Instantiated through Reflection.")] + private sealed class Expected + { + public string CrossVersionFixed { get; set; } + + public string CrossVersionNullFixed { get; set; } + + public string CrossVersionVariable { get; set; } + + public string CrossVersionNullVariable { get; set; } + + public string CrossVersionSparse { get; set; } + + public string CrossVersionNullSparse { get; set; } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerExampleUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerExampleUnitTests.cs new file mode 100644 index 0000000..f18ced6 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerExampleUnitTests.cs @@ -0,0 +1,481 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + // ReSharper disable once StringLiteralTypo + [TestClass] + [SuppressMessage("Naming", "DontUseVarForVariableTypes", Justification = "The types here are anonymous.")] + [DeploymentItem(@"TestData\CustomerSchema.json", "TestData")] + public sealed class CustomerExampleUnitTests + { + private readonly Hotel hotelExample = new Hotel() + { + Id = "The-Westin-St-John-Resort-Villas-1187", + Name = "The Westin St. John Resort Villas", + Phone = "+1 340-693-8000", + Address = new Address + { + Street = "300B Chocolate Hole", + City = "Great Cruz Bay", + State = "VI", + PostalCode = new PostalCode + { + Zip = 00830, + Plus4 = 0001, + }, + }, + }; + + private Namespace customerSchema; + private LayoutResolver customerResolver; + private Layout hotelLayout; + private Layout guestLayout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(@"TestData\CustomerSchema.json"); + this.customerSchema = Namespace.Parse(json); + this.customerResolver = new LayoutResolverNamespace(this.customerSchema); + this.hotelLayout = this.customerResolver.Resolve(this.customerSchema.Schemas.Find(x => x.Name == "Hotels").SchemaId); + this.guestLayout = this.customerResolver.Resolve(this.customerSchema.Schemas.Find(x => x.Name == "Guests").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateHotel() + { + RowBuffer row = new RowBuffer(0); + row.InitLayout(HybridRowVersion.V1, this.hotelLayout, this.customerResolver); + + Hotel h1 = this.hotelExample; + RowCursor root = RowCursor.Create(ref row); + this.WriteHotel(ref row, ref root, h1); + + root = RowCursor.Create(ref row); + Hotel h2 = this.ReadHotel(ref row, ref root); + + Assert.AreEqual(h1, h2); + } + + [TestMethod] + [Owner("jthunter")] + public void FrozenHotel() + { + RowBuffer row = new RowBuffer(0); + row.InitLayout(HybridRowVersion.V1, this.hotelLayout, this.customerResolver); + + Hotel h1 = this.hotelExample; + RowCursor root = RowCursor.Create(ref row); + this.WriteHotel(ref row, ref root, h1); + + root = RowCursor.Create(ref row); + ResultAssert.InsufficientPermissions(this.PartialUpdateHotelAddress(ref row, ref root, new Address { Street = "300B Brownie Way" })); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateGuest() + { + RowBuffer row = new RowBuffer(1024 * 1024); + row.InitLayout(HybridRowVersion.V1, this.guestLayout, this.customerResolver); + + Guest g1 = new Guest() + { + Id = Guid.Parse("64d9d6d3-fd6b-4556-8c6e-d960a7ece7b9"), + FirstName = "John", + LastName = "Adams", + Title = "President of the United States", + PhoneNumbers = new List { "(202) 456-1111" }, + ConfirmNumber = "(202) 456-1111", + Emails = new SortedSet { "president@whitehouse.gov" }, + Addresses = new Dictionary + { + ["home"] = new Address + { + Street = "1600 Pennsylvania Avenue NW", + City = "Washington, D.C.", + State = "DC", + PostalCode = new PostalCode + { + Zip = 20500, + Plus4 = 0001, + }, + }, + }, + }; + + RowCursor rc1 = RowCursor.Create(ref row); + this.WriteGuest(ref row, ref rc1, g1); + RowCursor rc2 = RowCursor.Create(ref row); + Guest g2 = this.ReadGuest(ref row, ref rc2); + Assert.AreEqual(g1, g2); + + // Append an item to an existing list. + RowCursor rc3 = RowCursor.Create(ref row); + int index = this.AppendGuestEmail(ref row, ref rc3, "vice_president@whitehouse.gov"); + Assert.AreEqual(1, index); + g1.Emails.Add("vice_president@whitehouse.gov"); + RowCursor rc4 = RowCursor.Create(ref row); + g2 = this.ReadGuest(ref row, ref rc4); + Assert.AreEqual(g1, g2); + + // Prepend an item to an existing list. + RowCursor rc5 = RowCursor.Create(ref row); + index = this.PrependGuestEmail(ref row, ref rc5, "ex_president@whitehouse.gov"); + Assert.AreEqual(0, index); + g1.Emails = new SortedSet { "ex_president@whitehouse.gov", "president@whitehouse.gov", "vice_president@whitehouse.gov" }; + RowCursor rc6 = RowCursor.Create(ref row); + g2 = this.ReadGuest(ref row, ref rc6); + Assert.AreEqual(g1, g2); + + // InsertAt an item to an existing list. + RowCursor rc7 = RowCursor.Create(ref row); + index = this.InsertAtGuestEmail(ref row, ref rc7, 1, "future_president@whitehouse.gov"); + Assert.AreEqual(1, index); + g1.Emails = new SortedSet + { + "ex_president@whitehouse.gov", + "future_president@whitehouse.gov", + "president@whitehouse.gov", + "vice_president@whitehouse.gov", + }; + + RowCursor rc8 = RowCursor.Create(ref row); + g2 = this.ReadGuest(ref row, ref rc8); + Assert.AreEqual(g1, g2); + } + + private static Address ReadAddress(ref RowBuffer row, ref RowCursor addressScope) + { + Address a = new Address(); + Layout addressLayout = addressScope.Layout; + Assert.IsTrue(addressLayout.TryFind("street", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref addressScope, c, out a.Street)); + Assert.IsTrue(addressLayout.TryFind("city", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref addressScope, c, out a.City)); + Assert.IsTrue(addressLayout.TryFind("state", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref addressScope, c, out a.State)); + + Assert.IsTrue(addressLayout.TryFind("postal_code", out c)); + addressScope.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref addressScope, out RowCursor postalCodeScope)); + a.PostalCode = CustomerExampleUnitTests.ReadPostalCode(ref row, ref postalCodeScope); + addressScope.Skip(ref row, ref postalCodeScope); + return a; + } + + private static PostalCode ReadPostalCode(ref RowBuffer row, ref RowCursor postalCodeScope) + { + Layout postalCodeLayout = postalCodeScope.Layout; + PostalCode pc = new PostalCode(); + Assert.IsTrue(postalCodeLayout.TryFind("zip", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref postalCodeScope, c, out pc.Zip)); + + Assert.IsTrue(postalCodeLayout.TryFind("plus4", out c)); + postalCodeScope.Find(ref row, c.Path); + if (c.TypeAs().ReadSparse(ref row, ref postalCodeScope, out short plus4) == Result.Success) + { + pc.Plus4 = plus4; + } + + return pc; + } + + private void WriteHotel(ref RowBuffer row, ref RowCursor root, Hotel h) + { + Assert.IsTrue(this.hotelLayout.TryFind("hotel_id", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, h.Id)); + Assert.IsTrue(this.hotelLayout.TryFind("name", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, h.Name)); + Assert.IsTrue(this.hotelLayout.TryFind("phone", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, h.Phone)); + + Assert.IsTrue(this.hotelLayout.TryFind("address", out c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref root, c.TypeArgs, out RowCursor addressScope)); + this.WriteAddress(ref row, ref addressScope, c.TypeArgs, h.Address); + root.Skip(ref row, ref addressScope); + } + + private void WriteGuest(ref RowBuffer row, ref RowCursor root, Guest g) + { + Assert.IsTrue(this.guestLayout.TryFind("guest_id", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref root, c, g.Id)); + Assert.IsTrue(this.guestLayout.TryFind("first_name", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, g.FirstName)); + Assert.IsTrue(this.guestLayout.TryFind("last_name", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, g.LastName)); + Assert.IsTrue(this.guestLayout.TryFind("title", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, g.Title)); + Assert.IsTrue(this.guestLayout.TryFind("confirm_number", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, g.ConfirmNumber)); + + if (g.Emails != null) + { + Assert.IsTrue(this.guestLayout.TryFind("emails", out c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref root, c.TypeArgs, out RowCursor emailScope)); + foreach (string email in g.Emails) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref emailScope, email)); + Assert.IsFalse(emailScope.MoveNext(ref row)); + } + + root.Skip(ref row, ref emailScope); + } + + if (g.PhoneNumbers != null) + { + Assert.IsTrue(this.guestLayout.TryFind("phone_numbers", out c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref root, c.TypeArgs, out RowCursor phoneNumbersScope)); + foreach (string phone in g.PhoneNumbers) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref phoneNumbersScope, phone)); + Assert.IsFalse(phoneNumbersScope.MoveNext(ref row)); + } + + root.Skip(ref row, ref phoneNumbersScope); + } + + if (g.Addresses != null) + { + Assert.IsTrue(this.guestLayout.TryFind("addresses", out c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref root, c.TypeArgs, out RowCursor addressesScope)); + TypeArgument tupleType = c.TypeAs().FieldType(ref addressesScope); + TypeArgument t0 = tupleType.TypeArgs[0]; + TypeArgument t1 = tupleType.TypeArgs[1]; + foreach (KeyValuePair pair in g.Addresses) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess( + tupleType.TypeAs().WriteScope(ref row, ref tempCursor, c.TypeArgs, out RowCursor tupleScope)); + + ResultAssert.IsSuccess(t0.TypeAs().WriteSparse(ref row, ref tupleScope, pair.Key)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + t1.TypeAs().WriteScope(ref row, ref tupleScope, t1.TypeArgs, out RowCursor addressScope)); + this.WriteAddress(ref row, ref addressScope, t1.TypeArgs, pair.Value); + Assert.IsFalse(tupleScope.MoveNext(ref row, ref addressScope)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref addressesScope, ref tempCursor)); + } + + root.Skip(ref row, ref addressesScope); + } + } + + private int AppendGuestEmail(ref RowBuffer row, ref RowCursor root, string email) + { + Assert.IsTrue(this.guestLayout.TryFind("emails", out LayoutColumn c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref root, out RowCursor emailScope)); + Assert.IsFalse(emailScope.MoveTo(ref row, int.MaxValue)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref emailScope, email)); + return emailScope.Index; + } + + private int PrependGuestEmail(ref RowBuffer row, ref RowCursor root, string email) + { + Assert.IsTrue(this.guestLayout.TryFind("emails", out LayoutColumn c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref root, out RowCursor emailScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref emailScope, email, UpdateOptions.InsertAt)); + return emailScope.Index; + } + + private int InsertAtGuestEmail(ref RowBuffer row, ref RowCursor root, int i, string email) + { + Assert.IsTrue(this.guestLayout.TryFind("emails", out LayoutColumn c)); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref root, out RowCursor emailScope)); + Assert.IsTrue(emailScope.MoveTo(ref row, i)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref emailScope, email, UpdateOptions.InsertAt)); + return emailScope.Index; + } + + private void WriteAddress(ref RowBuffer row, ref RowCursor addressScope, TypeArgumentList typeArgs, Address a) + { + Layout addressLayout = this.customerResolver.Resolve(typeArgs.SchemaId); + Assert.IsTrue(addressLayout.TryFind("street", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref addressScope, c, a.Street)); + Assert.IsTrue(addressLayout.TryFind("city", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref addressScope, c, a.City)); + Assert.IsTrue(addressLayout.TryFind("state", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref addressScope, c, a.State)); + + Assert.IsTrue(addressLayout.TryFind("postal_code", out c)); + addressScope.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref addressScope, c.TypeArgs, out RowCursor postalCodeScope)); + this.WritePostalCode(ref row, ref postalCodeScope, c.TypeArgs, a.PostalCode); + addressScope.Skip(ref row, ref postalCodeScope); + } + + private Result PartialUpdateHotelAddress(ref RowBuffer row, ref RowCursor root, Address a) + { + Assert.IsTrue(this.hotelLayout.TryFind("address", out LayoutColumn c)); + root.Find(ref row, c.Path); + Result r = c.TypeAs().ReadScope(ref row, ref root, out RowCursor addressScope); + if (r != Result.Success) + { + return r; + } + + Layout addressLayout = addressScope.Layout; + if (a.Street != null) + { + Assert.IsTrue(addressLayout.TryFind("street", out c)); + r = c.TypeAs().WriteVariable(ref row, ref addressScope, c, a.Street); + if (r != Result.Success) + { + return r; + } + } + + if (a.City != null) + { + Assert.IsTrue(addressLayout.TryFind("city", out c)); + r = c.TypeAs().WriteVariable(ref row, ref addressScope, c, a.City); + if (r != Result.Success) + { + return r; + } + } + + if (a.State != null) + { + Assert.IsTrue(addressLayout.TryFind("state", out c)); + r = c.TypeAs().WriteFixed(ref row, ref addressScope, c, a.State); + if (r != Result.Success) + { + return r; + } + } + + if (a.PostalCode != null) + { + Assert.IsTrue(addressLayout.TryFind("postal_code", out c)); + addressScope.Find(ref row, c.Path); + r = c.TypeAs().WriteScope(ref row, ref addressScope, c.TypeArgs, out RowCursor postalCodeScope); + if (r != Result.Success) + { + return r; + } + + this.WritePostalCode(ref row, ref postalCodeScope, c.TypeArgs, a.PostalCode); + } + + return Result.Success; + } + + private void WritePostalCode(ref RowBuffer row, ref RowCursor postalCodeScope, TypeArgumentList typeArgs, PostalCode pc) + { + Layout postalCodeLayout = this.customerResolver.Resolve(typeArgs.SchemaId); + Assert.IsNotNull(postalCodeLayout); + Assert.IsTrue(postalCodeLayout.TryFind("zip", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref postalCodeScope, c, pc.Zip)); + if (pc.Plus4.HasValue) + { + Assert.IsTrue(postalCodeLayout.TryFind("plus4", out c)); + postalCodeScope.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteSparse(ref row, ref postalCodeScope, pc.Plus4.Value)); + } + } + + private Hotel ReadHotel(ref RowBuffer row, ref RowCursor root) + { + Hotel h = new Hotel(); + Assert.IsTrue(this.hotelLayout.TryFind("hotel_id", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out h.Id)); + Assert.IsTrue(this.hotelLayout.TryFind("name", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out h.Name)); + Assert.IsTrue(this.hotelLayout.TryFind("phone", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out h.Phone)); + + Assert.IsTrue(this.hotelLayout.TryFind("address", out c)); + Assert.IsTrue(c.Type.Immutable); + root.Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref root, out RowCursor addressScope)); + Assert.IsTrue(addressScope.Immutable); + h.Address = CustomerExampleUnitTests.ReadAddress(ref row, ref addressScope); + root.Skip(ref row, ref addressScope); + return h; + } + + private Guest ReadGuest(ref RowBuffer row, ref RowCursor root) + { + Guest g = new Guest(); + Assert.IsTrue(this.guestLayout.TryFind("guest_id", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref root, c, out g.Id)); + Assert.IsTrue(this.guestLayout.TryFind("first_name", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out g.FirstName)); + Assert.IsTrue(this.guestLayout.TryFind("last_name", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out g.LastName)); + Assert.IsTrue(this.guestLayout.TryFind("title", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out g.Title)); + Assert.IsTrue(this.guestLayout.TryFind("confirm_number", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out g.ConfirmNumber)); + + Assert.IsTrue(this.guestLayout.TryFind("emails", out c)); + root.Clone(out RowCursor emailScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref emailScope, out emailScope) == Result.Success) + { + g.Emails = new SortedSet(); + while (emailScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref emailScope, out string item)); + g.Emails.Add(item); + } + } + + Assert.IsTrue(this.guestLayout.TryFind("phone_numbers", out c)); + root.Clone(out RowCursor phoneNumbersScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref phoneNumbersScope, out phoneNumbersScope) == Result.Success) + { + g.PhoneNumbers = new List(); + while (phoneNumbersScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref phoneNumbersScope, out string item)); + g.PhoneNumbers.Add(item); + } + } + + Assert.IsTrue(this.guestLayout.TryFind("addresses", out c)); + root.Clone(out RowCursor addressesScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref addressesScope, out addressesScope) == Result.Success) + { + TypeArgument tupleType = LayoutType.TypedMap.FieldType(ref addressesScope); + TypeArgument t0 = tupleType.TypeArgs[0]; + TypeArgument t1 = tupleType.TypeArgs[1]; + g.Addresses = new Dictionary(); + RowCursor pairScope = default; + while (addressesScope.MoveNext(ref row, ref pairScope)) + { + ResultAssert.IsSuccess(tupleType.TypeAs().ReadScope(ref row, ref addressesScope, out pairScope)); + Assert.IsTrue(pairScope.MoveNext(ref row)); + ResultAssert.IsSuccess(t0.TypeAs().ReadSparse(ref row, ref pairScope, out string key)); + Assert.IsTrue(pairScope.MoveNext(ref row)); + ResultAssert.IsSuccess(t1.TypeAs().ReadScope(ref row, ref pairScope, out RowCursor addressScope)); + Address value = CustomerExampleUnitTests.ReadAddress(ref row, ref addressScope); + g.Addresses.Add(key, value); + Assert.IsFalse(pairScope.MoveNext(ref row, ref addressScope)); + } + } + + return g; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Address.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Address.cs new file mode 100644 index 0000000..908badd --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Address.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema +{ + internal sealed class Address + { + public string Street; + public string City; + public string State; + public PostalCode PostalCode; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Address && this.Equals((Address)obj); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = this.Street != null ? this.Street.GetHashCode() : 0; + hashCode = (hashCode * 397) ^ (this.City != null ? this.City.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.State != null ? this.State.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.PostalCode != null ? this.PostalCode.GetHashCode() : 0); + return hashCode; + } + } + + private bool Equals(Address other) + { + return string.Equals(this.Street, other.Street) && + string.Equals(this.City, other.City) && + string.Equals(this.State, other.State) && + object.Equals(this.PostalCode, other.PostalCode); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/AddressSerializer.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/AddressSerializer.cs new file mode 100644 index 0000000..a43ea11 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/AddressSerializer.cs @@ -0,0 +1,104 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + internal static class AddressSerializer + { + public static Result Write(ref RowWriter writer, TypeArgument typeArg, Address obj) + { + Result r; + if (obj.Street != null) + { + r = writer.WriteString("street", obj.Street); + if (r != Result.Success) + { + return r; + } + } + + if (obj.City != null) + { + r = writer.WriteString("city", obj.City); + if (r != Result.Success) + { + return r; + } + } + + if (obj.State != null) + { + r = writer.WriteString("state", obj.State); + if (r != Result.Success) + { + return r; + } + } + + if (obj.PostalCode != null) + { + r = writer.WriteScope("postal_code", PostalCodeSerializer.TypeArg, obj.PostalCode, PostalCodeSerializer.Write); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + public static Result Read(ref RowReader reader, out Address obj) + { + obj = new Address(); + while (reader.Read()) + { + Result r; + switch (reader.Path) + { + case "street": + r = reader.ReadString(out obj.Street); + if (r != Result.Success) + { + return r; + } + + break; + case "city": + r = reader.ReadString(out obj.City); + if (r != Result.Success) + { + return r; + } + + break; + case "state": + r = reader.ReadString(out obj.State); + if (r != Result.Success) + { + return r; + } + + break; + case "postal_code": + r = reader.ReadScope( + obj, + (ref RowReader child, Address parent) => + PostalCodeSerializer.Read(ref child, out parent.PostalCode)); + + if (r != Result.Success) + { + return r; + } + + break; + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Guest.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Guest.cs new file mode 100644 index 0000000..a76a5e1 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Guest.cs @@ -0,0 +1,103 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema +{ + using System; + using System.Collections.Generic; + using System.Linq; + + internal sealed class Guest + { + public Guid Id; + public string FirstName; + public string LastName; + public string Title; + public ISet Emails; + public IList PhoneNumbers; + public IDictionary Addresses; + public string ConfirmNumber; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Guest && this.Equals((Guest)obj); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = this.Id.GetHashCode(); + hashCode = (hashCode * 397) ^ (this.FirstName != null ? this.FirstName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.LastName != null ? this.LastName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.Title != null ? this.Title.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.Emails != null ? this.Emails.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.PhoneNumbers != null ? this.PhoneNumbers.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.Addresses != null ? this.Addresses.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (this.ConfirmNumber != null ? this.ConfirmNumber.GetHashCode() : 0); + return hashCode; + } + } + + private static bool DictionaryEquals(IDictionary left, IDictionary right) + { + if (left == right) + { + return true; + } + + if ((left == null) || (right == null)) + { + return false; + } + + if (left.Count != right.Count) + { + return false; + } + + foreach (KeyValuePair p in left) + { + TValue value; + if (!right.TryGetValue(p.Key, out value)) + { + return false; + } + + if (!p.Value.Equals(value)) + { + return false; + } + } + + return true; + } + + private bool Equals(Guest other) + { + return this.Id.Equals(other.Id) && + string.Equals(this.FirstName, other.FirstName) && + string.Equals(this.LastName, other.LastName) && + string.Equals(this.Title, other.Title) && + string.Equals(this.ConfirmNumber, other.ConfirmNumber) && + ((this.Emails == other.Emails) || + ((this.Emails != null) && (other.Emails != null) && this.Emails.SetEquals(other.Emails))) && + ((this.PhoneNumbers == other.PhoneNumbers) || + ((this.PhoneNumbers != null) && (other.PhoneNumbers != null) && this.PhoneNumbers.SequenceEqual(other.PhoneNumbers))) && + Guest.DictionaryEquals(this.Addresses, other.Addresses); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Hotel.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Hotel.cs new file mode 100644 index 0000000..2377fc6 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/Hotel.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema +{ + internal sealed class Hotel + { + public string Id; + public string Name; + public string Phone; + public Address Address; + + public bool Equals(Hotel other) + { + return string.Equals(this.Id, other.Id) && + string.Equals(this.Name, other.Name) && + string.Equals(this.Phone, other.Phone) && + object.Equals(this.Address, other.Address); + } + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Hotel && this.Equals((Hotel)obj); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = this.Id?.GetHashCode() ?? 0; + hashCode = (hashCode * 397) ^ (this.Name?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Phone?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Address?.GetHashCode() ?? 0); + return hashCode; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/PostalCode.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/PostalCode.cs new file mode 100644 index 0000000..2edff67 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/PostalCode.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema +{ + internal sealed class PostalCode + { + public int Zip; + public short? Plus4; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is PostalCode && this.Equals((PostalCode)obj); + } + + public override int GetHashCode() + { + unchecked + { + return (this.Zip * 397) ^ this.Plus4.GetHashCode(); + } + } + + private bool Equals(PostalCode other) + { + return this.Zip == other.Zip && this.Plus4 == other.Plus4; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/PostalCodeSerializer.cs b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/PostalCodeSerializer.cs new file mode 100644 index 0000000..11b07ef --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/CustomerSchema/PostalCodeSerializer.cs @@ -0,0 +1,68 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + internal static class PostalCodeSerializer + { + public static TypeArgument TypeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(new SchemaId(1))); + + public static Result Write(ref RowWriter writer, TypeArgument typeArg, PostalCode obj) + { + Result r; + r = writer.WriteInt32("zip", obj.Zip); + if (r != Result.Success) + { + return r; + } + + if (obj.Plus4.HasValue) + { + r = writer.WriteInt16("plus4", obj.Plus4.Value); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + public static Result Read(ref RowReader reader, out PostalCode obj) + { + obj = new PostalCode(); + while (reader.Read()) + { + Result r; + switch (reader.Path) + { + case "zip": + r = reader.ReadInt32(out obj.Zip); + if (r != Result.Success) + { + return r; + } + + break; + case "plus4": + r = reader.ReadInt16(out short value); + if (r != Result.Success) + { + return r; + } + + obj.Plus4 = value; + break; + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/GlobalSuppressions.cs b/dotnet/src/HybridRow.Tests.Unit/GlobalSuppressions.cs new file mode 100644 index 0000000..ca4810f --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/GlobalSuppressions.cs @@ -0,0 +1,9 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +[assembly: + System.Diagnostics.CodeAnalysis.SuppressMessage( + "StyleCop.CSharp.LayoutRules", + "SA1515:Single-line comment should be preceded by blank line", + Justification = "Refactor")] diff --git a/dotnet/src/HybridRow.Tests.Unit/Internal/MurmurHash3UnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/Internal/MurmurHash3UnitTests.cs new file mode 100644 index 0000000..70b3fc9 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/Internal/MurmurHash3UnitTests.cs @@ -0,0 +1,102 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.Internal +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Internal; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class MurmurHash3UnitTests + { + private static readonly (ulong low, ulong high)[] Expected = new[] + { + (0x56F1549659CBEE1AUL, 0xCEB3EE124C3E3855UL), + (0xFE84B58886F9D717UL, 0xD24C5DE024F5EA6BUL), + (0x89F6250648BB11BFUL, 0x95595FB9D4CF58B0UL), + (0xC76AFDB39EDC6262UL, 0xB9286AF4FADAF497UL), + (0xC2CB4D9B3C9C247EUL, 0xB465D40116B8B7A2UL), + (0x317178F5B26D0B35UL, 0x1D564F53E2E468ADUL), + (0xE8D75F7C05F43F09UL, 0xA81CEA052AE92D6FUL), + (0x8F837665508C08A8UL, 0x2A74E6E47E5497BCUL), + (0x609778FDA1AFD731UL, 0x3EB1A0E3BFC653E4UL), + (0x0F59B8965FA49D1AUL, 0xCB3BC158243A5DEEUL), + (0x7A6D0AC9C98F5908UL, 0xBC93D3042C3E7178UL), + (0x863FE5AEBA9A3DFAUL, 0xDF42416658CB87C5UL), + (0xDB4C82337C8FB216UL, 0xCA7616B64ABF6B3DUL), + (0x0049223177425B48UL, 0x25510D7246BC3C2CUL), + (0x31AC129B24F82CABUL, 0xCD7174C2040E9834UL), + (0xCE39465288116345UL, 0x1CE6A26BA2E9E67DUL), + (0xD2BE55791E13DB17UL, 0xCF30BF3D93B3A9FAUL), + (0x43E323DD0F079145UL, 0xF06721555571ABBAUL), + (0xB0CE9F170A96F5BCUL, 0x18EE95960369D702UL), + (0xBFFAF6BEBC84A2A9UL, 0xE0612B6FC0C9D502UL), + (0x33E2D699697BC2DAUL, 0xB7E9CD6313DE05EEUL), + (0xCBFD7D8DA2A962BFUL, 0xCF4C281A7750E88AUL), + (0xBD8D863F83863088UL, 0x01AFFBDE3D405D35UL), + (0xBA2E05DF3328C7DBUL, 0x9620867ADDFE6579UL), + (0xC57BD1FB63CA0947UL, 0xE1391F8454D4EA9FUL), + (0x6AB710460A5BF9BAUL, 0x11D7E13FBEF63775UL), + (0x55C2C7C95F41C483UL, 0xA4DCC9F547A89563UL), + (0x8AA5A2031027F216UL, 0x1653FC7AD6CC6104UL), + (0xAD8A899FF093D9A5UL, 0x0EB26F6D1CCEB258UL), + (0xA3B6D57EBEB965D1UL, 0xE8078FCC5D8C2E3EUL), + (0x91ABF587B38224F6UL, 0x35899665A8A9252CUL), + (0xF05B1AF0487EE2D4UL, 0x5D7496C1665DDE12UL), + }; + + [TestMethod] + [Owner("jthunter")] + public void Hash128Check() + { + // Generate deterministic data for which the MurmurHash3 is known (see Expected). + Random rand = new Random(42); + byte[][] samples = new byte[MurmurHash3UnitTests.Expected.Length][]; + for (int i = 0; i < samples.Length; i++) + { + int sampleLength = rand.Next(10 * 1024); + samples[i] = new byte[sampleLength]; + rand.NextBytes(samples[i]); + } + + // Warm up the loop and verify correctness. + for (int i = 0; i < samples.Length; i++) + { + byte[] sample = samples[i]; + (ulong low, ulong high) = MurmurHash3.Hash128(sample, (0, 0)); + Console.WriteLine($"(0x{high:X16}UL, 0x{low:X16}UL),"); + Assert.AreEqual(MurmurHash3UnitTests.Expected[i].high, high); + Assert.AreEqual(MurmurHash3UnitTests.Expected[i].low, low); + } + + // Measure performance. + long ticks = MurmurHash3UnitTests.MeasureLoop(samples); + Console.WriteLine($"MurmurHash3: {ticks}"); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + [SuppressMessage("Microsoft.Reliability", "CA2001:Avoid calling problematic methods", Justification = "Perf Benchmark")] + private static long MeasureLoop(byte[][] samples) + { + const int outerLoopCount = 10000; + System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch(); + + GC.Collect(); + watch.Start(); + for (int j = 0; j < outerLoopCount; j++) + { + foreach (byte[] sample in samples) + { + MurmurHash3.Hash128(sample, (0, 0)); + } + } + + watch.Stop(); + return watch.ElapsedTicks; + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/LayoutCompilerUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/LayoutCompilerUnitTests.cs new file mode 100644 index 0000000..53d2f57 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/LayoutCompilerUnitTests.cs @@ -0,0 +1,1999 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +// ReSharper disable CommentTypo +// ReSharper disable StringLiteralTypo +#pragma warning disable SA1201 // Elements should appear in the correct order +#pragma warning disable SA1401 // Fields should be private +#pragma warning disable IDE0008 // Use explicit type + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.Linq; + using System.Text; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [SuppressMessage("Naming", "DontUseVarForVariableTypes", Justification = "The types here are anonymous.")] + public class LayoutCompilerUnitTests + { + private const int InitialRowSize = 2 * 1024 * 1024; + + [TestMethod] + [Owner("jthunter")] + public void PackNullAndBoolBits() + { + // Test that null bits and bool bits are packed tightly in the layout. + Schema s = new Schema { Name = "TestSchema", SchemaId = new SchemaId(1), Type = TypeKind.Schema }; + for (int i = 0; i < 32; i++) + { + s.Properties.Add( + new Property + { + Path = i.ToString(), + PropertyType = new PrimitivePropertyType { Type = TypeKind.Boolean, Storage = StorageKind.Fixed }, + }); + + Layout layout = s.Compile(new Namespace { Schemas = new List { s } }); + Assert.IsTrue(layout.Size == LayoutBit.DivCeiling((i + 1) * 2, LayoutType.BitsPerByte), "Size: {0}, i: {1}", layout.Size, i); + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaFixed() + { + // Test all fixed column types. + RoundTripFixed.Expected[] expectedSchemas = new[] + { + new RoundTripFixed.Expected { TypeName = "null", Default = default(NullValue), Value = NullValue.Default }, + new RoundTripFixed.Expected { TypeName = "bool", Default = default(bool), Value = false }, + + new RoundTripFixed.Expected { TypeName = "int8", Default = default(sbyte), Value = (sbyte)42 }, + new RoundTripFixed.Expected { TypeName = "int16", Default = default(short), Value = (short)42 }, + new RoundTripFixed.Expected { TypeName = "int32", Default = default(int), Value = 42 }, + new RoundTripFixed.Expected { TypeName = "int64", Default = default(long), Value = 42L }, + new RoundTripFixed.Expected { TypeName = "uint8", Default = default(byte), Value = (byte)42 }, + new RoundTripFixed.Expected { TypeName = "uint16", Default = default(ushort), Value = (ushort)42 }, + new RoundTripFixed.Expected { TypeName = "uint32", Default = default(uint), Value = 42U }, + new RoundTripFixed.Expected { TypeName = "uint64", Default = default(ulong), Value = 42UL }, + + new RoundTripFixed.Expected { TypeName = "float32", Default = default(float), Value = 4.2F }, + new RoundTripFixed.Expected { TypeName = "float64", Default = default(double), Value = 4.2 }, + new RoundTripFixed.Expected { TypeName = "float128", Default = default(Float128), Value = new Float128(0, 42) }, + new RoundTripFixed.Expected { TypeName = "decimal", Default = default(decimal), Value = 4.2M }, + + new RoundTripFixed.Expected { TypeName = "datetime", Default = default(DateTime), Value = DateTime.UtcNow }, + new RoundTripFixed.Expected { TypeName = "unixdatetime", Default = default(UnixDateTime), Value = new UnixDateTime(42) }, + + new RoundTripFixed.Expected { TypeName = "guid", Default = default(Guid), Value = Guid.NewGuid() }, + new RoundTripFixed.Expected { TypeName = "mongodbobjectid", Default = default(MongoDbObjectId), Value = new MongoDbObjectId(0, 42) }, + + new RoundTripFixed.Expected { TypeName = "utf8", Default = "\0\0", Value = "AB", Length = 2 }, + new RoundTripFixed.Expected + { + TypeName = "binary", + Default = new byte[] { 0x00, 0x00 }, + Value = new byte[] { 0x01, 0x02 }, + Length = 2, + }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (string nullable in new[] { "true", "false" }) + { + foreach (RoundTripFixed.Expected exp in expectedSchemas) + { + RoundTripFixed.Expected expected = exp; + string typeSchema = $@"{{'type': '{expected.TypeName}', 'storage': 'fixed', 'length': {expected.Length}, 'nullable': {nullable}}}"; + expected.Json = typeSchema; + string propSchema = $@"{{'path': 'a', 'type': {typeSchema}}}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{propSchema}] }}"; + try + { + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + Assert.AreEqual(1, layout.Columns.Length, "Json: {0}", expected.Json); + Assert.AreEqual(s.Name, layout.Name, "Json: {0}", expected.Json); + Assert.IsTrue(layout.ToString().Length > 0, "Json: {0}", expected.Json); + bool found = layout.TryFind("a", out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Fixed, col.Storage, "Json: {0}", expected.Json); + Assert.AreEqual(expected.Length == 0, col.Type.IsFixed, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + col.Type.LayoutCode, + ref row, + ref root, + new RoundTripFixed.Closure { Col = col, Expected = expected }); + } + catch (LayoutCompilationException) + { + Assert.AreEqual(expected.TypeName, "null"); + Assert.AreEqual("false", nullable); + } + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaVariable() + { + // Helper functions to create sample arrays. + string MakeS(int size) + { + StringBuilder ret = new StringBuilder(size); + for (int i = 0; i < size; i++) + { + ret.Append(unchecked((char)('a' + (i % 26)))); // allow wrapping (this also allows \0 chars) + } + + return ret.ToString(); + } + + byte[] MakeB(int size) + { + byte[] ret = new byte[size]; + for (int i = 0; i < size; i++) + { + ret[i] = unchecked((byte)(i + 1)); // allow wrapping + } + + return ret; + } + + // Test all variable column types. + RoundTripVariable.Expected[] expectedSchemas = new[] + { + new RoundTripVariable.Expected + { + Json = @"{'type': 'utf8', 'storage': 'variable', 'length': 100}", + Short = MakeS(2), + Value = MakeS(20), + Long = MakeS(100), + TooBig = MakeS(200), + }, + new RoundTripVariable.Expected + { + Json = @"{'type': 'binary', 'storage': 'variable', 'length': 100}", + Short = MakeB(2), + Value = MakeB(20), + Long = MakeB(100), + TooBig = MakeB(200), + }, + new RoundTripVariable.Expected { Json = @"{'type': 'varint', 'storage': 'variable'}", Short = 1L, Value = 255L, Long = long.MaxValue }, + new RoundTripVariable.Expected { Json = @"{'type': 'varuint', 'storage': 'variable'}", Short = 1UL, Value = 255UL, Long = ulong.MaxValue }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripVariable.Expected expected in expectedSchemas) + { + string propSchema = $@"{{'path': 'a', 'type': {expected.Json}}}, + {{'path': 'b', 'type': {expected.Json}}}, + {{'path': 'c', 'type': {expected.Json}}}"; + + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{propSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + bool found = layout.TryFind("a", out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.IsTrue(col.Type.AllowVariable); + Assert.AreEqual(StorageKind.Variable, col.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref root, new RoundTripVariable.Closure { Layout = layout, Col = col, Expected = expected }); + } + } + + [TestMethod] + [Owner("jthunter")] + public void SparseOrdering() + { + // Test various orderings of multiple sparse column types. + RoundTripSparseOrdering.Expected[][] expectedOrders = new[] + { + new[] + { + new RoundTripSparseOrdering.Expected { Path = "a", Type = LayoutType.Utf8, Value = "aa" }, + new RoundTripSparseOrdering.Expected { Path = "b", Type = LayoutType.Utf8, Value = "bb" }, + }, + new[] + { + new RoundTripSparseOrdering.Expected { Path = "a", Type = LayoutType.VarInt, Value = 42L }, + new RoundTripSparseOrdering.Expected { Path = "b", Type = LayoutType.Int64, Value = 43L }, + }, + new[] + { + new RoundTripSparseOrdering.Expected { Path = "a", Type = LayoutType.VarInt, Value = 42L }, + new RoundTripSparseOrdering.Expected { Path = "b", Type = LayoutType.Utf8, Value = "aa" }, + new RoundTripSparseOrdering.Expected { Path = "c", Type = LayoutType.Null, Value = NullValue.Default }, + new RoundTripSparseOrdering.Expected { Path = "d", Type = LayoutType.Boolean, Value = true }, + }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseOrdering.Expected[] expectedSet in expectedOrders) + { + foreach (IEnumerable permutation in expectedSet.Permute()) + { + string json = string.Join(", ", from p in permutation select p.Path + ": " + p.Type.Name); + Console.WriteLine("{0}", json); + + row.Reset(); + row.InitLayout(HybridRowVersion.V1, Layout.Empty, SystemSchema.LayoutResolver); + foreach (RoundTripSparseOrdering.Expected field in permutation) + { + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + field.Type.LayoutCode, ref row, ref root, new RoundTripSparseOrdering.Closure { Expected = field, Json = json }); + } + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaSparseSimple() + { + // Test all sparse column types. + RoundTripSparseSimple.Expected[] expectedSchemas = new[] + { + new RoundTripSparseSimple.Expected { Json = @"{'type': 'null', 'storage': 'sparse'}", Value = NullValue.Default }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'bool', 'storage': 'sparse'}", Value = true }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'bool', 'storage': 'sparse'}", Value = false }, + + new RoundTripSparseSimple.Expected { Json = @"{'type': 'int8', 'storage': 'sparse'}", Value = (sbyte)42 }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'int16', 'storage': 'sparse'}", Value = (short)42 }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'int32', 'storage': 'sparse'}", Value = 42 }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'int64', 'storage': 'sparse'}", Value = 42L }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'uint8', 'storage': 'sparse'}", Value = (byte)42 }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'uint16', 'storage': 'sparse'}", Value = (ushort)42 }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'uint32', 'storage': 'sparse'}", Value = 42U }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'uint64', 'storage': 'sparse'}", Value = 42UL }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'varint', 'storage': 'sparse'}", Value = 42L }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'varuint', 'storage': 'sparse'}", Value = 42UL }, + + new RoundTripSparseSimple.Expected { Json = @"{'type': 'float32', 'storage': 'sparse'}", Value = 4.2F }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'float64', 'storage': 'sparse'}", Value = 4.2 }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'float128', 'storage': 'sparse'}", Value = new Float128(0, 42) }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'decimal', 'storage': 'sparse'}", Value = 4.2M }, + + new RoundTripSparseSimple.Expected { Json = @"{'type': 'datetime', 'storage': 'sparse'}", Value = DateTime.UtcNow }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'unixdatetime', 'storage': 'sparse'}", Value = new UnixDateTime(42) }, + + new RoundTripSparseSimple.Expected { Json = @"{'type': 'guid', 'storage': 'sparse'}", Value = Guid.NewGuid() }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'mongodbobjectid', 'storage': 'sparse'}", Value = new MongoDbObjectId(0, 42) }, + + new RoundTripSparseSimple.Expected { Json = @"{'type': 'utf8', 'storage': 'sparse'}", Value = "AB" }, + new RoundTripSparseSimple.Expected { Json = @"{'type': 'binary', 'storage': 'sparse'}", Value = new byte[] { 0x01, 0x02 } }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseSimple.Expected expected in expectedSchemas) + { + string propSchema = $@"{{'path': 'a', 'type': {expected.Json}}}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{propSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + Assert.AreEqual(1, layout.Columns.Length, "Json: {0}", expected.Json); + bool found = layout.TryFind("a", out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, col.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref root, new RoundTripSparseSimple.Closure { Col = col, Expected = expected }); + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaUDT() + { + string namespaceJson = @" + {'name': 'myNamespace', + 'schemas': [ + {'name': 'udtA', 'id': 1, 'type': 'schema', 'options': { 'disallowUnschematized': false }, + 'properties': [ + { 'path': 'a', 'type': { 'type': 'int8', 'storage': 'fixed' }}, + { 'path': 'b', 'type': { 'type': 'utf8', 'storage': 'variable', 'length': 100 }} + ] + }, + {'name': 'udtB', 'id': 2, 'type': 'schema'}, + {'name': 'udtB', 'id': 3, 'type': 'schema'}, + {'name': 'udtB', 'id': 4, 'type': 'schema'}, + {'name': 'table', 'id': -1, 'type': 'schema', + 'properties': [ + { 'path': 'u', 'type': { 'type': 'schema', 'name': 'udtA' }}, + { 'path': 'v', 'type': { 'type': 'schema', 'name': 'udtB', 'id': 3 }}, + ] + } + ] + }"; + + Namespace n1 = Namespace.Parse(namespaceJson); + + string tag = $"Json: {namespaceJson}"; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + Schema s = n1.Schemas.Find(x => x.Name == "table"); + Assert.IsNotNull(s); + Assert.AreEqual("table", s.Name); + Layout layout = s.Compile(n1); + bool found = layout.TryFind("u", out LayoutColumn udtACol); + Assert.IsTrue(found, tag); + Assert.AreEqual(StorageKind.Sparse, udtACol.Storage, tag); + + Schema udtASchema = n1.Schemas.Find(x => x.SchemaId == udtACol.TypeArgs.SchemaId); + Assert.IsNotNull(udtASchema); + Assert.AreEqual("udtA", udtASchema.Name); + + // Verify that UDT versioning works through schema references. + found = layout.TryFind("v", out LayoutColumn udtBCol); + Assert.IsTrue(found, tag); + Assert.AreEqual(StorageKind.Sparse, udtBCol.Storage, tag); + Schema udtBSchema = n1.Schemas.Find(x => x.SchemaId == udtBCol.TypeArgs.SchemaId); + Assert.IsNotNull(udtBSchema); + Assert.AreEqual("udtB", udtBSchema.Name); + Assert.AreEqual(new SchemaId(3), udtBSchema.SchemaId); + + LayoutResolver resolver = new LayoutResolverNamespace(n1); + Layout udtLayout = resolver.Resolve(udtASchema.SchemaId); + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + // Verify the udt doesn't yet exist. + RowCursor.Create(ref row, out RowCursor scope).Find(ref row, udtACol.Path); + Result r = LayoutType.UDT.ReadScope(ref row, ref scope, out _); + ResultAssert.NotFound(r, tag); + r = LayoutType.UDT.WriteScope(ref row, ref scope, udtACol.TypeArgs, out RowCursor udtScope1); + ResultAssert.IsSuccess(r, tag); + r = LayoutType.UDT.ReadScope(ref row, ref scope, out RowCursor udtScope2); + ResultAssert.IsSuccess(r, tag); + Assert.AreSame(udtLayout, udtScope2.Layout, tag); + Assert.AreEqual(udtScope1.ScopeType, udtScope2.ScopeType, tag); + Assert.AreEqual(udtScope1.start, udtScope2.start, tag); + Assert.AreEqual(udtScope1.Immutable, udtScope2.Immutable, tag); + + var expectedSchemas = new[] + { + new + { + Storage = StorageKind.Fixed, + Path = "a", + FixedExpected = new RoundTripFixed.Expected { Json = @"{ 'type': 'int8', 'storage': 'fixed' }", Value = (sbyte)42 }, + VariableExpected = default(RoundTripVariable.Expected), + }, + new + { + Storage = StorageKind.Variable, + Path = "b", + FixedExpected = default(RoundTripFixed.Expected), + VariableExpected = new RoundTripVariable.Expected { Json = @"{ 'type': 'utf8', 'storage': 'variable' }", Value = "AB" }, + }, + }; + + foreach (var expected in expectedSchemas) + { + found = udtLayout.TryFind(expected.Path, out LayoutColumn col); + Assert.IsTrue(found, "Path: {0}", expected.Path); + StorageKind storage = expected.Storage; + switch (storage) + { + case StorageKind.Fixed: + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref udtScope1, new RoundTripFixed.Closure + { + Col = col, + Expected = expected.FixedExpected, + }); + break; + case StorageKind.Variable: + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref udtScope1, new RoundTripVariable.Closure + { + Col = col, Layout = layout, + Expected = expected.VariableExpected, + }); + break; + } + } + + RowCursor.Create(ref row).AsReadOnly(out RowCursor roRoot).Find(ref row, udtACol.Path); + ResultAssert.InsufficientPermissions(udtACol.TypeAs().DeleteScope(ref row, ref roRoot)); + ResultAssert.InsufficientPermissions(udtACol.TypeAs().WriteScope(ref row, ref roRoot, udtACol.TypeArgs, out udtScope2)); + + // Overwrite the whole scope. + RowCursor.Create(ref row, out scope).Find(ref row, udtACol.Path); + r = LayoutType.Null.WriteSparse(ref row, ref scope, NullValue.Default); + ResultAssert.IsSuccess(r, tag); + r = LayoutType.UDT.ReadScope(ref row, ref scope, out RowCursor _); + ResultAssert.TypeMismatch(r, tag); + r = LayoutType.UDT.DeleteScope(ref row, ref scope); + ResultAssert.TypeMismatch(r, tag); + + // Overwrite it again, then delete it. + RowCursor.Create(ref row, out scope).Find(ref row, udtACol.Path); + r = LayoutType.UDT.WriteScope(ref row, ref scope, udtACol.TypeArgs, out RowCursor _); + ResultAssert.IsSuccess(r, tag); + r = LayoutType.UDT.DeleteScope(ref row, ref scope); + ResultAssert.IsSuccess(r, tag); + RowCursor.Create(ref row, out scope).Find(ref row, udtACol.Path); + r = LayoutType.UDT.ReadScope(ref row, ref scope, out RowCursor _); + ResultAssert.NotFound(r, tag); + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaSparseObject() + { + // Test all fixed column types. + RoundTripSparseObject.Expected[] expectedSchemas = new[] + { + new RoundTripSparseObject.Expected { Json = @"{'path': 'b', 'type': {'type': 'int8'}}", Value = (sbyte)42 }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseObject.Expected expected in expectedSchemas) + { + string objectColumnSchema = $"{{'path': 'a', 'type': {{'type': 'object', 'properties': [{expected.Json}] }} }}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{objectColumnSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + Assert.AreEqual(1, layout.Columns.Length, "Json: {0}", expected.Json); + bool found = layout.TryFind("a", out LayoutColumn objCol); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, objCol.Storage, "Json: {0}", expected.Json); + found = layout.TryFind("a.b", out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, col.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref root, new RoundTripSparseObject.Closure + { + ObjCol = objCol, + Col = col, + Expected = expected, + }); + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaSparseObjectMulti() + { + // Test sparse object columns with various kinds of sparse column fields. + RoundTripSparseObjectMulti.Expected[] expectedSchemas = new[] + { + new RoundTripSparseObjectMulti.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'int8'}}", + Props = new[] + { + new RoundTripSparseObjectMulti.Property { Path = "a.b", Value = (sbyte)42 }, + }, + }, + new RoundTripSparseObjectMulti.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'int8'}}, {'path': 'c', 'type': {'type': 'utf8'}}", + Props = new[] + { + new RoundTripSparseObjectMulti.Property { Path = "a.b", Value = (sbyte)42 }, + new RoundTripSparseObjectMulti.Property { Path = "a.c", Value = "abc" }, + }, + }, + new RoundTripSparseObjectMulti.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'int8'}}, + {'path': 'c', 'type': {'type': 'bool'}}, + {'path': 'd', 'type': {'type': 'binary'}}, + {'path': 'e', 'type': {'type': 'null'}}", + Props = new[] + { + new RoundTripSparseObjectMulti.Property { Path = "a.b", Value = (sbyte)42 }, + new RoundTripSparseObjectMulti.Property { Path = "a.c", Value = true }, + new RoundTripSparseObjectMulti.Property { Path = "a.d", Value = new byte[] { 0x01, 0x02, 0x03 } }, + new RoundTripSparseObjectMulti.Property { Path = "a.e", Value = NullValue.Default }, + }, + }, + new RoundTripSparseObjectMulti.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'object'}}", + Props = new[] + { + new RoundTripSparseObjectMulti.Property { Path = "a.b" }, + }, + }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseObjectMulti.Expected expected in expectedSchemas) + { + string objectColumnSchema = $"{{'path': 'a', 'type': {{'type': 'object', 'properties': [{expected.Json}] }} }}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{objectColumnSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + Assert.AreEqual(1, layout.Columns.Length, "Json: {0}", expected.Json); + bool found = layout.TryFind("a", out LayoutColumn objCol); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, objCol.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + // Verify the object doesn't exist. + LayoutObject objT = objCol.Type as LayoutObject; + Assert.IsNotNull(objT); + RowCursor.Create(ref row, out RowCursor field).Find(ref row, objCol.Path); + Result r = objT.ReadScope(ref row, ref field, out RowCursor scope); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + + // Write the object and the nested column. + r = objT.WriteScope(ref row, ref field, objCol.TypeArgs, out scope); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + + foreach (IEnumerable permutation in expected.Props.Permute()) + { + foreach (RoundTripSparseObjectMulti.Property prop in permutation) + { + found = layout.TryFind(prop.Path, out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, col.Storage, "Json: {0}", expected.Json); + + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref scope, new RoundTripSparseObjectMulti.Closure + { + Col = col, + Prop = prop, + Expected = expected, + }); + } + } + + // Write something after the scope. + UtfAnyString otherColumnPath = "not-" + objCol.Path; + field.Clone(out RowCursor otherColumn).Find(ref row, otherColumnPath); + r = LayoutType.Boolean.WriteSparse(ref row, ref otherColumn, true); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + + // Overwrite the whole scope. + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = objT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + + // Read the thing after the scope and verify it is still there. + field.Clone(out otherColumn).Find(ref row, otherColumnPath); + r = LayoutType.Boolean.ReadSparse(ref row, ref otherColumn, out bool notScope); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + Assert.IsTrue(notScope); + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaSparseObjectNested() + { + // Test nested sparse object columns with various kinds of sparse column fields. + RoundTripSparseObjectNested.Expected[] expectedSchemas = new[] + { + new RoundTripSparseObjectNested.Expected + { + Json = @"{'path': 'c', 'type': {'type': 'int8'}}", + Props = new[] + { + new RoundTripSparseObjectNested.Property { Path = "a.b.c", Value = (sbyte)42 }, + }, + }, + new RoundTripSparseObjectNested.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'int8'}}, {'path': 'c', 'type': {'type': 'utf8'}}", + Props = new[] + { + new RoundTripSparseObjectNested.Property { Path = "a.b.b", Value = (sbyte)42 }, + new RoundTripSparseObjectNested.Property { Path = "a.b.c", Value = "abc" }, + }, + }, + new RoundTripSparseObjectNested.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'int8'}}, + {'path': 'c', 'type': {'type': 'bool'}}, + {'path': 'd', 'type': {'type': 'binary'}}, + {'path': 'e', 'type': {'type': 'null'}}", + Props = new[] + { + new RoundTripSparseObjectNested.Property { Path = "a.b.b", Value = (sbyte)42 }, + new RoundTripSparseObjectNested.Property { Path = "a.b.c", Value = true }, + new RoundTripSparseObjectNested.Property { Path = "a.b.d", Value = new byte[] { 0x01, 0x02, 0x03 } }, + new RoundTripSparseObjectNested.Property { Path = "a.b.e", Value = NullValue.Default }, + }, + }, + new RoundTripSparseObjectNested.Expected + { + Json = @"{'path': 'b', 'type': {'type': 'object'}}", + Props = new[] + { + new RoundTripSparseObjectNested.Property { Path = "a.b.b" }, + }, + }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseObjectNested.Expected expected in expectedSchemas) + { + string nestedColumnSchema = $"{{'path': 'b', 'type': {{'type': 'object', 'properties': [{expected.Json}] }} }}"; + string objectColumnSchema = $"{{'path': 'a', 'type': {{'type': 'object', 'properties': [{nestedColumnSchema}] }} }}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{objectColumnSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + bool found = layout.TryFind("a", out LayoutColumn objCol); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, objCol.Storage, "Json: {0}", expected.Json); + found = layout.TryFind("a.b", out LayoutColumn objCol2); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, objCol2.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + RowCursor root = RowCursor.Create(ref row); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + // Write the object. + LayoutObject objT = objCol.Type as LayoutObject; + Assert.IsNotNull(objT); + root.Clone(out RowCursor field).Find(ref row, objCol.Path); + Result r = objT.WriteScope(ref row, ref field, objCol.TypeArgs, out RowCursor _); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + + foreach (IEnumerable permutation in expected.Props.Permute()) + { + foreach (RoundTripSparseObjectNested.Property prop in permutation) + { + found = layout.TryFind(prop.Path, out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, col.Storage, "Json: {0}", expected.Json); + + this.LayoutCodeSwitch( + col.Type.LayoutCode, ref row, ref root, new RoundTripSparseObjectNested.Closure + { + Col = col, + Prop = prop, + Expected = expected, + }); + } + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaSparseArray() + { + // Test all fixed column types. + RoundTripSparseArray.Expected[] expectedSchemas = new[] + { + new RoundTripSparseArray.Expected + { + Json = @"array[null]", + Type = LayoutType.Null, + Value = new List { NullValue.Default, NullValue.Default, NullValue.Default }, + }, + new RoundTripSparseArray.Expected { Json = @"array[bool]", Type = LayoutType.Boolean, Value = new List { true, false, true } }, + + new RoundTripSparseArray.Expected { Json = @"array[int8]", Type = LayoutType.Int8, Value = new List { (sbyte)42, (sbyte)43, (sbyte)44 } }, + new RoundTripSparseArray.Expected { Json = @"array[int16]", Type = LayoutType.Int16, Value = new List { (short)42, (short)43, (short)44 } }, + new RoundTripSparseArray.Expected { Json = @"array[int32]", Type = LayoutType.Int32, Value = new List { 42, 43, 44 } }, + new RoundTripSparseArray.Expected { Json = @"array[int64]", Type = LayoutType.Int64, Value = new List { 42L, 43L, 44L } }, + new RoundTripSparseArray.Expected { Json = @"array[uint8]", Type = LayoutType.UInt8, Value = new List { (byte)42, (byte)43, (byte)44 } }, + new RoundTripSparseArray.Expected { Json = @"array[uint16]", Type = LayoutType.UInt16, Value = new List { (ushort)42, (ushort)43, (ushort)44 } }, + new RoundTripSparseArray.Expected { Json = @"array[uint32]", Type = LayoutType.UInt32, Value = new List { 42u, 43u, 44u } }, + new RoundTripSparseArray.Expected { Json = @"array[uint64]", Type = LayoutType.UInt64, Value = new List { 42UL, 43UL, 44UL } }, + + new RoundTripSparseArray.Expected { Json = @"array[varint]", Type = LayoutType.VarInt, Value = new List { 42L, 43L, 44L } }, + new RoundTripSparseArray.Expected { Json = @"array[varuint]", Type = LayoutType.VarUInt, Value = new List { 42UL, 43UL, 44UL } }, + + new RoundTripSparseArray.Expected { Json = @"array[float32]", Type = LayoutType.Float32, Value = new List { 4.2F, 4.3F, 4.4F } }, + new RoundTripSparseArray.Expected { Json = @"array[float64]", Type = LayoutType.Float64, Value = new List { 4.2, 4.3, 4.4 } }, + new RoundTripSparseArray.Expected + { + Json = @"array[float128]", + Type = LayoutType.Float128, + Value = new List { new Float128(0, 42), new Float128(0, 43), new Float128(0, 44) }, + }, + new RoundTripSparseArray.Expected { Json = @"array[decimal]", Type = LayoutType.Decimal, Value = new List { 4.2M, 4.3M, 4.4M } }, + + new RoundTripSparseArray.Expected + { + Json = @"array[datetime]", + Type = LayoutType.DateTime, + Value = new List { DateTime.UtcNow, DateTime.UtcNow.AddTicks(1), DateTime.UtcNow.AddTicks(2) }, + }, + new RoundTripSparseArray.Expected + { + Json = @"array[unixdatetime]", + Type = LayoutType.UnixDateTime, + Value = new List { new UnixDateTime(1), new UnixDateTime(2), new UnixDateTime(3) }, + }, + new RoundTripSparseArray.Expected + { + Json = @"array[guid]", + Type = LayoutType.Guid, + Value = new List { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() }, + }, + new RoundTripSparseArray.Expected + { + Json = @"array[mongodbobjectid]", + Type = LayoutType.MongoDbObjectId, + Value = new List { new MongoDbObjectId(0, 1), new MongoDbObjectId(0, 2), new MongoDbObjectId(0, 3) }, + }, + + new RoundTripSparseArray.Expected { Json = @"array[utf8]", Type = LayoutType.Utf8, Value = new List { "abc", "def", "xyz" } }, + new RoundTripSparseArray.Expected + { + Json = @"array[binary]", + Type = LayoutType.Binary, + Value = new List { new byte[] { 0x01, 0x02 }, new byte[] { 0x03, 0x04 }, new byte[] { 0x05, 0x06 } }, + }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseArray.Expected expected in expectedSchemas) + { + foreach (Type arrT in new[] { typeof(LayoutTypedArray), typeof(LayoutArray) }) + { + string arrayColumnSchema = @"{'path': 'a', 'type': {'type': 'array', 'items': {'type': 'any'}} }"; + if (arrT == typeof(LayoutTypedArray)) + { + arrayColumnSchema = $@"{{'path': 'a', 'type': {{'type': 'array', + 'items': {{'type': '{expected.Type.Name}', 'nullable': false }}}} }}"; + } + + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{arrayColumnSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + bool found = layout.TryFind("a", out LayoutColumn arrCol); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, arrCol.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + expected.Type.LayoutCode, ref row, ref root, new RoundTripSparseArray.Closure + { + ArrCol = arrCol, + Expected = expected, + }); + } + } + } + + [TestMethod] + [Owner("jthunter")] + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1139", Justification = "Need to control the binary ordering.")] + public void ParseSchemaSparseSet() + { + // Test all fixed column types. + RoundTripSparseSet.Expected[] expectedSchemas = new[] + { + new RoundTripSparseSet.Expected { Json = @"set[null]", Type = LayoutType.Null, Value = new List { NullValue.Default } }, + new RoundTripSparseSet.Expected { Json = @"set[bool]", Type = LayoutType.Boolean, Value = new List { false, true } }, + + new RoundTripSparseSet.Expected { Json = @"set[int8]", Type = LayoutType.Int8, Value = new List { (sbyte)42, (sbyte)43, (sbyte)44 } }, + + new RoundTripSparseSet.Expected { Json = @"set[int16]", Type = LayoutType.Int16, Value = new List { (short)42, (short)43, (short)44 } }, + new RoundTripSparseSet.Expected { Json = @"set[int32]", Type = LayoutType.Int32, Value = new List { 42, 43, 44 } }, + new RoundTripSparseSet.Expected { Json = @"set[int64]", Type = LayoutType.Int64, Value = new List { 42L, 43L, 44L } }, + new RoundTripSparseSet.Expected { Json = @"set[uint8]", Type = LayoutType.UInt8, Value = new List { (byte)42, (byte)43, (byte)44 } }, + new RoundTripSparseSet.Expected { Json = @"set[uint16]", Type = LayoutType.UInt16, Value = new List { (ushort)42, (ushort)43, (ushort)44 } }, + new RoundTripSparseSet.Expected { Json = @"set[uint32]", Type = LayoutType.UInt32, Value = new List { 42u, 43u, 44u } }, + new RoundTripSparseSet.Expected { Json = @"set[uint64]", Type = LayoutType.UInt64, Value = new List { 42UL, 43UL, 44UL } }, + + new RoundTripSparseSet.Expected { Json = @"set[varint]", Type = LayoutType.VarInt, Value = new List { 42L, 43L, 44L } }, + new RoundTripSparseSet.Expected { Json = @"set[varuint]", Type = LayoutType.VarUInt, Value = new List { 42UL, 43UL, 44UL } }, + + new RoundTripSparseSet.Expected { Json = @"set[float32]", Type = LayoutType.Float32, Value = new List { 4.2F, 4.3F, 4.4F } }, + new RoundTripSparseSet.Expected + { + Json = @"set[float64]", + Type = LayoutType.Float64, + Value = new List + { + (double)0xAAAAAAAAAAAAAAAA, + (double)0xBBBBBBBBBBBBBBBB, + (double)0xCCCCCCCCCCCCCCCC, + }, + }, + new RoundTripSparseSet.Expected { Json = @"set[decimal]", Type = LayoutType.Decimal, Value = new List { 4.2M, 4.3M, 4.4M } }, + + new RoundTripSparseSet.Expected + { + Json = @"set[datetime]", + Type = LayoutType.DateTime, + Value = new List + { + new DateTime(1, DateTimeKind.Unspecified), + new DateTime(2, DateTimeKind.Unspecified), + new DateTime(3, DateTimeKind.Unspecified), + }, + }, + new RoundTripSparseSet.Expected + { + Json = @"set[guid]", + Type = LayoutType.Guid, + Value = new List + { + Guid.Parse("AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"), + Guid.Parse("BBBBBBBB-BBBB-BBBB-BBBB-BBBBBBBBBBBB"), + Guid.Parse("CCCCCCCC-CCCC-CCCC-CCCC-CCCCCCCCCCCC"), + }, + }, + + new RoundTripSparseSet.Expected { Json = @"set[utf8]", Type = LayoutType.Utf8, Value = new List { "abc", "def", "xyz" } }, + new RoundTripSparseSet.Expected + { + Json = @"set[binary]", + Type = LayoutType.Binary, + Value = new List { new byte[] { 0x01, 0x02 }, new byte[] { 0x03, 0x04 }, new byte[] { 0x05, 0x06 } }, + }, + }; + + RowBuffer row = new RowBuffer(LayoutCompilerUnitTests.InitialRowSize); + foreach (RoundTripSparseSet.Expected expected in expectedSchemas) + { + foreach (Type setT in new[] { typeof(LayoutTypedSet) }) + { + string setColumnSchema = @"{'path': 'a', 'type': {'type': 'set', 'items': {'type': 'any'}} }"; + if (setT == typeof(LayoutTypedSet)) + { + setColumnSchema = $@"{{'path': 'a', 'type': {{'type': 'set', + 'items': {{'type': '{expected.Type.Name}', 'nullable': false }}}} }}"; + } + + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{setColumnSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(new Namespace { Schemas = new List { s } }); + Layout layout = resolver.Resolve(new SchemaId(-1)); + bool found = layout.TryFind("a", out LayoutColumn setCol); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.AreEqual(StorageKind.Sparse, setCol.Storage, "Json: {0}", expected.Json); + + // Try writing a row using the layout. + row.Reset(); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + + HybridRowHeader header = row.Header; + Assert.AreEqual(HybridRowVersion.V1, header.Version); + Assert.AreEqual(layout.SchemaId, header.SchemaId); + + RowCursor root = RowCursor.Create(ref row); + this.LayoutCodeSwitch( + expected.Type.LayoutCode, ref row, ref root, new RoundTripSparseSet.Closure + { + SetCol = setCol, + Expected = expected, + }); + } + } + } + + /// Ensure that a parent scope exists in the row. + /// The row to create the desired scope. + /// The root scope. + /// The scope to create. + /// A string to tag errors with. + /// The enclosing scope. + private static RowCursor EnsureScope(ref RowBuffer row, ref RowCursor root, LayoutColumn col, string tag) + { + if (col == null) + { + return root; + } + + RowCursor parentScope = LayoutCompilerUnitTests.EnsureScope(ref row, ref root, col.Parent, tag); + + LayoutObject pT = col.Type as LayoutObject; + Assert.IsNotNull(pT); + parentScope.Clone(out RowCursor field).Find(ref row, col.Path); + Result r = pT.ReadScope(ref row, ref field, out RowCursor scope); + if (r == Result.NotFound) + { + r = pT.WriteScope(ref row, ref field, col.TypeArgs, out scope); + } + + ResultAssert.IsSuccess(r, tag); + return scope; + } + + private void LayoutCodeSwitch(LayoutCode code, ref RowBuffer row, ref RowCursor scope, TClosure closure) + where TDispatcher : TestActionDispatcher, new() + { + TDispatcher dispatcher = new TDispatcher(); + switch (code) + { + case LayoutCode.Null: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Boolean: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Int8: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Int16: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Int32: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Int64: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.UInt8: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.UInt16: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.UInt32: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.UInt64: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.VarInt: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.VarUInt: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Float32: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Float64: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Float128: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Decimal: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.DateTime: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.UnixDateTime: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Guid: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.MongoDbObjectId: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Utf8: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.Binary: + dispatcher.Dispatch(ref row, ref scope, closure); + break; + case LayoutCode.ObjectScope: + dispatcher.DispatchObject(ref row, ref scope, closure); + break; + default: + Contract.Assert(false, $"Unknown type will be ignored: {code}"); + break; + } + } + + private sealed class RoundTripFixed : TestActionDispatcher + { + public struct Expected + { + public string TypeName; + public string Json; + public object Value; + public object Default; + public int Length; + } + + public struct Closure + { + public LayoutColumn Col; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn col = closure.Col; + Expected expected = closure.Expected; + Result r; + TValue value; + + Console.WriteLine("{0}", expected.Json); + TLayout t = (TLayout)col.Type; + if (col.NullBit != LayoutBit.Invalid) + { + r = t.ReadFixed(ref row, ref root, col, out value); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + } + else + { + r = t.ReadFixed(ref row, ref root, col, out value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + if (expected.Default is Array defaultArray) + { + CollectionAssert.AreEqual(defaultArray, (ICollection)value, "Json: {0}", expected.Json); + } + else + { + Assert.AreEqual(expected.Default, value, "Json: {0}", expected.Json); + } + } + + r = t.WriteFixed(ref row, ref root, col, (TValue)expected.Value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.ReadFixed(ref row, ref root, col, out value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + if (expected.Value is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, "Json: {0}", expected.Json); + } + else + { + Assert.AreEqual(expected.Value, value, "Json: {0}", expected.Json); + } + + root.AsReadOnly(out RowCursor roRoot); + ResultAssert.InsufficientPermissions(t.WriteFixed(ref row, ref roRoot, col, (TValue)expected.Value)); + ResultAssert.InsufficientPermissions(t.DeleteFixed(ref row, ref roRoot, col)); + + if (col.NullBit != LayoutBit.Invalid) + { + ResultAssert.IsSuccess(t.DeleteFixed(ref row, ref root, col)); + } + else + { + ResultAssert.TypeMismatch(t.DeleteFixed(ref row, ref root, col)); + } + } + } + + private class RoundTripVariable : TestActionDispatcher + { + public struct Expected + { + public string Json; + public object Short; + public object Value; + public object Long; + public object TooBig; + } + + public struct Closure + { + public LayoutColumn Col; + public Layout Layout; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn col = closure.Col; + Expected expected = closure.Expected; + + Console.WriteLine("{0}", expected.Json); + + this.RoundTrip(ref row, ref root, col, expected.Value, expected); + } + + protected void RoundTrip( + ref RowBuffer row, + ref RowCursor root, + LayoutColumn col, + object exValue, + Expected expected) + where TLayout : LayoutType + { + TLayout t = (TLayout)col.Type; + Result r = t.WriteVariable(ref row, ref root, col, (TValue)exValue); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + this.Compare(ref row, ref root, col, exValue, expected); + + root.AsReadOnly(out RowCursor roRoot); + ResultAssert.InsufficientPermissions(t.WriteVariable(ref row, ref roRoot, col, (TValue)expected.Value)); + } + + protected void Compare( + ref RowBuffer row, + ref RowCursor root, + LayoutColumn col, + object exValue, + Expected expected) + where TLayout : LayoutType + { + TLayout t = (TLayout)col.Type; + Result r = t.ReadVariable(ref row, ref root, col, out TValue value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + if (exValue is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, "Json: {0}", expected.Json); + } + else + { + Assert.AreEqual(exValue, value, "Json: {0}", expected.Json); + } + } + } + + private sealed class VariableInterleaving : RoundTripVariable + { + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + Layout layout = closure.Layout; + Expected expected = closure.Expected; + + Console.WriteLine("{0}", expected.Json); + + LayoutColumn a = this.Verify(ref row, ref root, layout, "a", expected); + LayoutColumn b = this.Verify(ref row, ref root, layout, "b", expected); + LayoutColumn c = this.Verify(ref row, ref root, layout, "c", expected); + + this.RoundTrip(ref row, ref root, b, expected.Value, expected); + this.RoundTrip(ref row, ref root, a, expected.Value, expected); + this.RoundTrip(ref row, ref root, c, expected.Value, expected); + + // Make the var column shorter. + int rowSizeBeforeShrink = row.Length; + this.RoundTrip(ref row, ref root, a, expected.Short, expected); + this.Compare(ref row, ref root, c, expected.Value, expected); + int rowSizeAfterShrink = row.Length; + Assert.IsTrue(rowSizeAfterShrink < rowSizeBeforeShrink, "Json: {0}", expected.Json); + + // Make the var column longer. + this.RoundTrip(ref row, ref root, a, expected.Long, expected); + this.Compare(ref row, ref root, c, expected.Value, expected); + int rowSizeAfterGrow = row.Length; + Assert.IsTrue(rowSizeAfterGrow > rowSizeAfterShrink, "Json: {0}", expected.Json); + Assert.IsTrue(rowSizeAfterGrow > rowSizeBeforeShrink, "Json: {0}", expected.Json); + + // Check for size overflow errors. + if (a.Size > 0) + { + this.TooBig(ref row, ref root, a, expected); + } + + // Delete the var column. + this.Delete(ref row, ref root, b, expected); + this.Delete(ref row, ref root, c, expected); + this.Delete(ref row, ref root, a, expected); + } + + private LayoutColumn Verify(ref RowBuffer row, ref RowCursor root, Layout layout, string path, Expected expected) + where TLayout : LayoutType + { + bool found = layout.TryFind(path, out LayoutColumn col); + Assert.IsTrue(found, "Json: {0}", expected.Json); + Assert.IsTrue(col.Type.AllowVariable); + TLayout t = (TLayout)col.Type; + Result r = t.ReadVariable(ref row, ref root, col, out TValue _); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + return col; + } + + private void TooBig(ref RowBuffer row, ref RowCursor root, LayoutColumn col, Expected expected) + where TLayout : LayoutType + { + TLayout t = (TLayout)col.Type; + Result r = t.WriteVariable(ref row, ref root, col, (TValue)expected.TooBig); + Assert.AreEqual(Result.TooBig, r, "Json: {0}", expected.Json); + } + + private void Delete(ref RowBuffer row, ref RowCursor root, LayoutColumn col, Expected expected) + where TLayout : LayoutType + { + TLayout t = (TLayout)col.Type; + root.AsReadOnly(out RowCursor roRoot); + ResultAssert.InsufficientPermissions(t.DeleteVariable(ref row, ref roRoot, col)); + Result r = t.DeleteVariable(ref row, ref root, col); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.ReadVariable(ref row, ref root, col, out TValue _); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + } + } + + private sealed class RoundTripSparseOrdering : TestActionDispatcher + { + public struct Expected + { + public string Path; + public LayoutType Type; + public object Value; + } + + public struct Closure + { + public string Json; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutType type = closure.Expected.Type; + string path = closure.Expected.Path; + object exValue = closure.Expected.Value; + string json = closure.Json; + + TLayout t = (TLayout)type; + TValue value = (TValue)exValue; + root.Clone(out RowCursor field).Find(ref row, path); + Result r = t.WriteSparse(ref row, ref field, value); + ResultAssert.IsSuccess(r, "Json: {0}", json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.IsSuccess(r, "Json: {0}", json); + if (exValue is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, "Json: {0}", json); + } + else + { + Assert.AreEqual(exValue, value, "Json: {0}", json); + } + + if (t is LayoutNull) + { + r = LayoutType.Boolean.WriteSparse(ref row, ref field, false); + ResultAssert.IsSuccess(r, "Json: {0}", json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.TypeMismatch(r, "Json: {0}", json); + } + else + { + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, "Json: {0}", json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.TypeMismatch(r, "Json: {0}", json); + } + } + } + + private sealed class RoundTripSparseSimple : TestActionDispatcher + { + public struct Expected + { + public string Json; + public object Value; + } + + public struct Closure + { + public LayoutColumn Col; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn col = closure.Col; + Expected expected = closure.Expected; + + Console.WriteLine("{0}", col.Type.Name); + TLayout t = (TLayout)col.Type; + root.Clone(out RowCursor field).Find(ref row, col.Path); + Result r = t.ReadSparse(ref row, ref field, out TValue value); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + r = t.WriteSparse(ref row, ref field, (TValue)expected.Value, UpdateOptions.Update); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + r = t.WriteSparse(ref row, ref field, (TValue)expected.Value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.WriteSparse(ref row, ref field, (TValue)expected.Value, UpdateOptions.Insert); + ResultAssert.Exists(r, "Json: {0}", expected.Json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + if (expected.Value is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, "Json: {0}", expected.Json); + } + else + { + Assert.AreEqual(expected.Value, value, "Json: {0}", expected.Json); + } + + root.AsReadOnly(out RowCursor roRoot).Find(ref row, col.Path); + ResultAssert.InsufficientPermissions(t.DeleteSparse(ref row, ref roRoot)); + ResultAssert.InsufficientPermissions(t.WriteSparse(ref row, ref roRoot, (TValue)expected.Value, UpdateOptions.Update)); + + if (t is LayoutNull) + { + r = LayoutType.Boolean.WriteSparse(ref row, ref field, false); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + } + else + { + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + } + + r = t.DeleteSparse(ref row, ref field); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + + // Overwrite it again, then delete it. + r = t.WriteSparse(ref row, ref field, (TValue)expected.Value, UpdateOptions.Update); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.DeleteSparse(ref row, ref field); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + } + } + + private sealed class RoundTripSparseObject : TestActionDispatcher + { + public struct Expected + { + public string Json; + public object Value; + } + + public struct Closure + { + public LayoutColumn ObjCol; + public LayoutColumn Col; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn objCol = closure.ObjCol; + LayoutObject objT = objCol.Type as LayoutObject; + LayoutColumn col = closure.Col; + Expected expected = closure.Expected; + + Console.WriteLine("{0}", col.Type.Name); + Assert.IsNotNull(objT, "Json: {0}", expected.Json); + Assert.AreEqual(objCol, col.Parent, "Json: {0}", expected.Json); + + TLayout t = (TLayout)col.Type; + + // Attempt to read the object and the nested column. + root.Clone(out RowCursor field).Find(ref row, objCol.Path); + Result r = objT.ReadScope(ref row, ref field, out RowCursor scope); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + + // Write the object and the nested column. + r = objT.WriteScope(ref row, ref field, objCol.TypeArgs, out scope); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + + // Verify the nested field doesn't yet appear within the new scope. + scope.Clone(out RowCursor nestedField).Find(ref row, col.Path); + r = t.ReadSparse(ref row, ref nestedField, out TValue value); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + + // Write the nested field. + r = t.WriteSparse(ref row, ref nestedField, (TValue)expected.Value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + + // Read the object and the nested column, validate the nested column has the proper value. + r = objT.ReadScope(ref row, ref field, out RowCursor scope2); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + Assert.AreEqual(scope.ScopeType, scope2.ScopeType, "Json: {0}", expected.Json); + Assert.AreEqual(scope.start, scope2.start, "Json: {0}", expected.Json); + Assert.AreEqual(scope.Immutable, scope2.Immutable, "Json: {0}", expected.Json); + + // Read the nested field + scope2.Clone(out nestedField).Find(ref row, col.Path); + r = t.ReadSparse(ref row, ref nestedField, out value); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + if (expected.Value is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, "Json: {0}", expected.Json); + } + else + { + Assert.AreEqual(expected.Value, value, "Json: {0}", expected.Json); + } + + root.AsReadOnly(out RowCursor roRoot).Find(ref row, objCol.Path); + ResultAssert.InsufficientPermissions(objT.DeleteScope(ref row, ref roRoot)); + ResultAssert.InsufficientPermissions(objT.WriteScope(ref row, ref roRoot, objCol.TypeArgs, out scope2)); + + // Overwrite the whole scope. + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = objT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + r = objT.DeleteScope(ref row, ref field); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + + // Overwrite it again, then delete it. + r = objT.WriteScope(ref row, ref field, objCol.TypeArgs, out RowCursor _, UpdateOptions.Update); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = objT.DeleteScope(ref row, ref field); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = objT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + } + } + + private sealed class RoundTripSparseObjectMulti : TestActionDispatcher + { + public struct Expected + { + public string Json; + public Property[] Props; + } + + public struct Property + { + public string Path; + public object Value; + } + + public struct Closure + { + public LayoutColumn Col; + public Property Prop; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor scope, Closure closure) + { + LayoutColumn col = closure.Col; + Property prop = closure.Prop; + Expected expected = closure.Expected; + string tag = string.Format("Prop: {1}: Json: {0}", expected.Json, prop.Path); + + Console.WriteLine(tag); + + TLayout t = (TLayout)col.Type; + + // Verify the nested field doesn't yet appear within the new scope. + scope.Clone(out RowCursor nestedField).Find(ref row, col.Path); + Result r = t.ReadSparse(ref row, ref nestedField, out TValue value); + Assert.IsTrue(r == Result.NotFound || r == Result.TypeMismatch, tag); + + // Write the nested field. + r = t.WriteSparse(ref row, ref nestedField, (TValue)prop.Value); + ResultAssert.IsSuccess(r, tag); + + // Read the nested field + r = t.ReadSparse(ref row, ref nestedField, out value); + ResultAssert.IsSuccess(r, tag); + if (prop.Value is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual(prop.Value, value, tag); + } + + // Overwrite the nested field. + if (t is LayoutNull) + { + r = LayoutType.Boolean.WriteSparse(ref row, ref nestedField, false); + ResultAssert.IsSuccess(r, tag); + } + else + { + r = LayoutType.Null.WriteSparse(ref row, ref nestedField, NullValue.Default); + ResultAssert.IsSuccess(r, tag); + } + + // Verify nested field no longer there. + r = t.ReadSparse(ref row, ref nestedField, out value); + ResultAssert.TypeMismatch(r, tag); + } + + public override void DispatchObject(ref RowBuffer row, ref RowCursor scope, Closure closure) + { + LayoutColumn col = closure.Col; + Property prop = closure.Prop; + Expected expected = closure.Expected; + string tag = string.Format("Prop: {1}: Json: {0}", expected.Json, prop.Path); + + Console.WriteLine(tag); + + LayoutObject t = (LayoutObject)col.Type; + + // Verify the nested field doesn't yet appear within the new scope. + scope.Clone(out RowCursor nestedField).Find(ref row, col.Path); + Result r = t.ReadScope(ref row, ref nestedField, out RowCursor scope2); + ResultAssert.NotFound(r, tag); + + // Write the nested field. + r = t.WriteScope(ref row, ref nestedField, col.TypeArgs, out scope2); + ResultAssert.IsSuccess(r, tag); + + // Read the nested field + r = t.ReadScope(ref row, ref nestedField, out RowCursor scope3); + ResultAssert.IsSuccess(r, tag); + Assert.AreEqual(scope2.AsReadOnly(out RowCursor _).ScopeType, scope3.ScopeType, tag); + Assert.AreEqual(scope2.AsReadOnly(out RowCursor _).start, scope3.start, tag); + + // Overwrite the nested field. + r = LayoutType.Null.WriteSparse(ref row, ref nestedField, NullValue.Default); + ResultAssert.IsSuccess(r, tag); + + // Verify nested field no longer there. + r = t.ReadScope(ref row, ref nestedField, out scope3); + ResultAssert.TypeMismatch(r, tag); + } + } + + private sealed class RoundTripSparseObjectNested : TestActionDispatcher + { + public struct Expected + { + public string Json; + public Property[] Props; + } + + public struct Property + { + public string Path; + public object Value; + } + + public struct Closure + { + public LayoutColumn Col; + public Property Prop; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn col = closure.Col; + Property prop = closure.Prop; + Expected expected = closure.Expected; + string tag = string.Format("Prop: {1}: Json: {0}", expected.Json, prop.Path); + + Console.WriteLine(tag); + + TLayout t = (TLayout)col.Type; + + // Ensure scope exists. + RowCursor scope = LayoutCompilerUnitTests.EnsureScope(ref row, ref root, col.Parent, tag); + + // Write the nested field. + scope.Clone(out RowCursor field).Find(ref row, col.Path); + Result r = t.WriteSparse(ref row, ref field, (TValue)prop.Value); + ResultAssert.IsSuccess(r, tag); + + // Read the nested field + r = t.ReadSparse(ref row, ref field, out TValue value); + ResultAssert.IsSuccess(r, tag); + if (prop.Value is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual(prop.Value, value, tag); + } + + // Overwrite the nested field. + if (t is LayoutNull) + { + r = LayoutType.Boolean.WriteSparse(ref row, ref field, false); + ResultAssert.IsSuccess(r, tag); + } + else + { + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, tag); + } + + // Verify nested field no longer there. + r = t.ReadSparse(ref row, ref field, out value); + ResultAssert.TypeMismatch(r, tag); + } + + public override void DispatchObject(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn col = closure.Col; + Property prop = closure.Prop; + Expected expected = closure.Expected; + string tag = string.Format("Prop: {1}: Json: {0}", expected.Json, prop.Path); + + Console.WriteLine(tag); + + // Ensure scope exists. + RowCursor scope = LayoutCompilerUnitTests.EnsureScope(ref row, ref root, col, tag); + Assert.AreNotEqual(root, scope); + } + } + + private sealed class RoundTripSparseArray : TestActionDispatcher + { + public struct Expected + { + public string Json; + public LayoutType Type; + public List Value; + } + + public struct Closure + { + public LayoutColumn ArrCol; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn arrCol = closure.ArrCol; + LayoutIndexedScope arrT = arrCol.Type as LayoutIndexedScope; + Expected expected = closure.Expected; + string tag = $"Json: {expected.Json}, Array: {arrCol.Type.Name}"; + + Console.WriteLine(tag); + Assert.IsNotNull(arrT, tag); + + TLayout t = (TLayout)expected.Type; + + // Verify the array doesn't yet exist. + root.Clone(out RowCursor field).Find(ref row, arrCol.Path); + Result r = arrT.ReadScope(ref row, ref field, out RowCursor scope); + ResultAssert.NotFound(r, tag); + + // Write the array. + r = arrT.WriteScope(ref row, ref field, arrCol.TypeArgs, out scope); + ResultAssert.IsSuccess(r, tag); + + // Verify the nested field doesn't yet appear within the new scope. + Assert.IsFalse(scope.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref scope, out TValue value); + ResultAssert.NotFound(r, tag); + + // Write the nested fields. + scope.Clone(out RowCursor elm); + foreach (object item in expected.Value) + { + // Write the ith index. + r = t.WriteSparse(ref row, ref elm, (TValue)item); + ResultAssert.IsSuccess(r, tag); + + // Move cursor to the ith+1 index. + Assert.IsFalse(elm.MoveNext(ref row)); + } + + // Read the array and the nested column, validate the nested column has the proper value. + r = arrT.ReadScope(ref row, ref field, out RowCursor scope2); + ResultAssert.IsSuccess(r, tag); + Assert.AreEqual(scope.ScopeType, scope2.ScopeType, tag); + Assert.AreEqual(scope.start, scope2.start, tag); + Assert.AreEqual(scope.Immutable, scope2.Immutable, tag); + + // Read the nested fields + scope2.Clone(out elm); + foreach (object item in expected.Value) + { + Assert.IsTrue(elm.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref elm, out value); + ResultAssert.IsSuccess(r, tag); + if (item is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual((TValue)item, value, tag); + } + } + + // Delete an item. + int indexToDelete = 1; + Assert.IsTrue(scope2.Clone(out elm).MoveTo(ref row, indexToDelete)); + r = t.DeleteSparse(ref row, ref elm); + ResultAssert.IsSuccess(r, tag); + List remainingValues = new List(expected.Value); + remainingValues.RemoveAt(indexToDelete); + scope2.Clone(out elm); + foreach (object item in remainingValues) + { + Assert.IsTrue(elm.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref elm, out value); + ResultAssert.IsSuccess(r, tag); + if (item is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual(item, value, tag); + } + } + + Assert.IsFalse(scope2.Clone(out elm).MoveTo(ref row, remainingValues.Count)); + + root.AsReadOnly(out RowCursor roRoot).Find(ref row, arrCol.Path); + ResultAssert.InsufficientPermissions(arrT.DeleteScope(ref row, ref roRoot)); + ResultAssert.InsufficientPermissions(arrT.WriteScope(ref row, ref roRoot, arrCol.TypeArgs, out scope2)); + + // Overwrite the whole scope. + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, tag); + r = arrT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.TypeMismatch(r, tag); + r = arrT.DeleteScope(ref row, ref field); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + + // Overwrite it again, then delete it. + r = arrT.WriteScope(ref row, ref field, arrCol.TypeArgs, out RowCursor _, UpdateOptions.Update); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = arrT.DeleteScope(ref row, ref field); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = arrT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + } + } + + private sealed class RoundTripSparseSet : TestActionDispatcher + { + public struct Expected + { + public string Json; + public LayoutType Type; + public List Value; + } + + public struct Closure + { + public LayoutColumn SetCol; + public Expected Expected; + } + + public override void Dispatch(ref RowBuffer row, ref RowCursor root, Closure closure) + { + LayoutColumn setCol = closure.SetCol; + LayoutUniqueScope setT = setCol.Type as LayoutUniqueScope; + Expected expected = closure.Expected; + string tag = $"Json: {expected.Json}, Set: {setCol.Type.Name}"; + + Console.WriteLine(tag); + Assert.IsNotNull(setT, tag); + + TLayout t = (TLayout)expected.Type; + + // Verify the Set doesn't yet exist. + root.Clone(out RowCursor field).Find(ref row, setCol.Path); + Result r = setT.ReadScope(ref row, ref field, out RowCursor scope); + ResultAssert.NotFound(r, tag); + + // Write the Set. + r = setT.WriteScope(ref row, ref field, setCol.TypeArgs, out scope); + ResultAssert.IsSuccess(r, tag); + + // Verify the nested field doesn't yet appear within the new scope. + Assert.IsFalse(scope.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref scope, out TValue value); + ResultAssert.NotFound(r, tag); + + // Write the nested fields. + foreach (object v1 in expected.Value) + { + // Write the ith item into staging storage. + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + r = t.WriteSparse(ref row, ref tempCursor, (TValue)v1); + ResultAssert.IsSuccess(r, tag); + + // Move item into the set. + r = setT.MoveField(ref row, ref scope, ref tempCursor); + ResultAssert.IsSuccess(r, tag); + } + + // Attempts to insert the same items into the set again will fail. + foreach (object v2 in expected.Value) + { + // Write the ith item into staging storage. + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + r = t.WriteSparse(ref row, ref tempCursor, (TValue)v2); + ResultAssert.IsSuccess(r, tag); + + // Move item into the set. + r = setT.MoveField(ref row, ref scope, ref tempCursor, UpdateOptions.Insert); + ResultAssert.Exists(r, tag); + } + + // Read the Set and the nested column, validate the nested column has the proper value. + r = setT.ReadScope(ref row, ref field, out RowCursor scope2); + ResultAssert.IsSuccess(r, tag); + Assert.AreEqual(scope.ScopeType, scope2.ScopeType, tag); + Assert.AreEqual(scope.start, scope2.start, tag); + Assert.AreEqual(scope.Immutable, scope2.Immutable, tag); + + // Read the nested fields + ResultAssert.IsSuccess(setT.ReadScope(ref row, ref field, out scope)); + foreach (object item in expected.Value) + { + Assert.IsTrue(scope.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref scope, out value); + ResultAssert.IsSuccess(r, tag); + if (item is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual(item, value, tag); + } + } + + // Delete all of the items and then insert them again in the opposite order. + ResultAssert.IsSuccess(setT.ReadScope(ref row, ref field, out scope)); + for (int i = 0; i < expected.Value.Count; i++) + { + Assert.IsTrue(scope.MoveNext(ref row)); + r = t.DeleteSparse(ref row, ref scope); + ResultAssert.IsSuccess(r, tag); + } + + ResultAssert.IsSuccess(setT.ReadScope(ref row, ref field, out scope)); + for (int i = expected.Value.Count - 1; i >= 0; i--) + { + // Write the ith item into staging storage. + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + r = t.WriteSparse(ref row, ref tempCursor, (TValue)expected.Value[i]); + ResultAssert.IsSuccess(r, tag); + + // Move item into the set. + r = setT.MoveField(ref row, ref scope, ref tempCursor); + ResultAssert.IsSuccess(r, tag); + } + + // Verify they still enumerate in sorted order. + ResultAssert.IsSuccess(setT.ReadScope(ref row, ref field, out scope)); + foreach (object item in expected.Value) + { + Assert.IsTrue(scope.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref scope, out value); + ResultAssert.IsSuccess(r, tag); + if (item is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual(item, value, tag); + } + } + + // Delete one item. + if (expected.Value.Count > 1) + { + int indexToDelete = 1; + ResultAssert.IsSuccess(setT.ReadScope(ref row, ref field, out scope)); + Assert.IsTrue(scope.MoveTo(ref row, indexToDelete)); + r = t.DeleteSparse(ref row, ref scope); + ResultAssert.IsSuccess(r, tag); + List remainingValues = new List(expected.Value); + remainingValues.RemoveAt(indexToDelete); + + ResultAssert.IsSuccess(setT.ReadScope(ref row, ref field, out scope)); + foreach (object item in remainingValues) + { + Assert.IsTrue(scope.MoveNext(ref row)); + r = t.ReadSparse(ref row, ref scope, out value); + ResultAssert.IsSuccess(r, tag); + if (item is Array array) + { + CollectionAssert.AreEqual(array, (ICollection)value, tag); + } + else + { + Assert.AreEqual(item, value, tag); + } + } + + Assert.IsFalse(scope.MoveTo(ref row, remainingValues.Count)); + } + + root.AsReadOnly(out RowCursor roRoot).Find(ref row, setCol.Path); + ResultAssert.InsufficientPermissions(setT.DeleteScope(ref row, ref roRoot)); + ResultAssert.InsufficientPermissions(setT.WriteScope(ref row, ref roRoot, setCol.TypeArgs, out _)); + + // Overwrite the whole scope. + r = LayoutType.Null.WriteSparse(ref row, ref field, NullValue.Default); + ResultAssert.IsSuccess(r, tag); + r = setT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.TypeMismatch(r, tag); + r = setT.DeleteScope(ref row, ref field); + ResultAssert.TypeMismatch(r, "Json: {0}", expected.Json); + + // Overwrite it again, then delete it. + r = setT.WriteScope(ref row, ref field, setCol.TypeArgs, out RowCursor _, UpdateOptions.Update); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = setT.DeleteScope(ref row, ref field); + ResultAssert.IsSuccess(r, "Json: {0}", expected.Json); + r = setT.ReadScope(ref row, ref field, out RowCursor _); + ResultAssert.NotFound(r, "Json: {0}", expected.Json); + } + } + + private abstract class TestActionDispatcher + { + public abstract void Dispatch(ref RowBuffer row, ref RowCursor scope, TClosure closure) + where TLayout : LayoutType; + + public virtual void DispatchObject(ref RowBuffer row, ref RowCursor scope, TClosure closure) + { + Assert.Fail("not implemented"); + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/LayoutTypeUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/LayoutTypeUnitTests.cs new file mode 100644 index 0000000..15aead6 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/LayoutTypeUnitTests.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class LayoutTypeUnitTests + { + [TestMethod] + [Owner("jthunter")] + public void LayoutTypeTest() + { + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Boolean); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Int8); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Int16); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Int32); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Int64); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.UInt8); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.UInt16); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.UInt32); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.UInt64); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.VarInt); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.VarUInt); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Float32); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Float64); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Decimal); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Null); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Boolean); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.DateTime); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Guid); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Utf8); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Binary); + LayoutTypeUnitTests.TestLayoutTypeApi(LayoutType.Object); + } + + private static void TestLayoutTypeApi(LayoutType t) + { + Assert.IsNotNull(t.Name); + Assert.IsFalse(string.IsNullOrWhiteSpace(t.Name)); + Assert.AreNotSame(null, t.IsFixed, t.Name); + Assert.AreNotSame(null, t.AllowVariable, t.Name); + Assert.AreNotSame(null, t.IsBool, t.Name); + Assert.AreNotSame(null, t.IsNull, t.Name); + Assert.AreNotSame(null, t.IsVarint, t.Name); + Assert.IsTrue(t.Size >= 0, t.Name); + Assert.AreNotEqual(LayoutCode.Invalid, t.LayoutCode, t.Name); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.csproj b/dotnet/src/HybridRow.Tests.Unit/Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.csproj new file mode 100644 index 0000000..ce8bea0 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.csproj @@ -0,0 +1,82 @@ + + + + true + true + {DC93CAA3-9732-46D4-ACBF-D69EFC3F6511} + Library + Test + Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit + Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit + netcoreapp2.2 + True + AnyCPU + + + + MsTest_Latest + FrameworkCore20 + X64 + $(OutDir) + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + diff --git a/dotnet/src/HybridRow.Tests.Unit/NullableUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/NullableUnitTests.cs new file mode 100644 index 0000000..d3cab60 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/NullableUnitTests.cs @@ -0,0 +1,437 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +// ReSharper disable StringLiteralTypo +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Linq; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(NullableUnitTests.SchemaFile, "TestData")] + public sealed class NullableUnitTests + { + private const string SchemaFile = @"TestData\NullableSchema.json"; + private const int InitialRowSize = 2 * 1024 * 1024; + + private Namespace schema; + private LayoutResolver resolver; + private Layout layout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(NullableUnitTests.SchemaFile); + this.schema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.schema); + this.layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Nullables").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateNullables() + { + RowBuffer row = new RowBuffer(NullableUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + Nullables t1 = new Nullables + { + NullBool = new List { true, false, null }, + NullArray = new List { 1.2F, null, 3.0F }, + NullSet = new List { null, "abc", "def" }, + NullTuple = new List<(int?, long?)> + { + (1, 2), (null, 3), (4, null), + (null, null), + }, + NullMap = new Dictionary + { + { Guid.Parse("{00000000-0000-0000-0000-000000000000}"), 1 }, + { Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), 20 }, + { Guid.Parse("{7499C40E-7077-45C1-AE5F-3E384966B3B9}"), null }, + }, + }; + + this.WriteNullables(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + Nullables t2 = this.ReadNullables(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(t1, t2); + } + + private static Result WriteNullable( + ref RowBuffer row, + ref RowCursor scope, + TypeArgument itemType, + TValue? item, + out RowCursor nullableScope) + where TValue : struct + { + return NullableUnitTests.WriteNullableImpl(ref row, ref scope, itemType, item.HasValue, item ?? default, out nullableScope); + } + + private static Result WriteNullable( + ref RowBuffer row, + ref RowCursor scope, + TypeArgument itemType, + TValue item, + out RowCursor nullableScope) + where TValue : class + { + return NullableUnitTests.WriteNullableImpl(ref row, ref scope, itemType, item != null, item, out nullableScope); + } + + private static Result WriteNullableImpl( + ref RowBuffer row, + ref RowCursor scope, + TypeArgument itemType, + bool hasValue, + TValue item, + out RowCursor nullableScope) + { + Result r = itemType.TypeAs() + .WriteScope(ref row, ref scope, itemType.TypeArgs, hasValue, out nullableScope); + + if (r != Result.Success) + { + return r; + } + + if (hasValue) + { + r = itemType.TypeArgs[0].Type.TypeAs>().WriteSparse(ref row, ref nullableScope, item); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + private static Result ReadNullable( + ref RowBuffer row, + ref RowCursor scope, + TypeArgument itemType, + out TValue? item, + out RowCursor nullableScope) + where TValue : struct + { + Result r = NullableUnitTests.ReadNullableImpl(ref row, ref scope, itemType, out TValue value, out nullableScope); + if ((r != Result.Success) && (r != Result.NotFound)) + { + item = null; + return r; + } + + item = (r == Result.NotFound) ? (TValue?)null : value; + return Result.Success; + } + + private static Result ReadNullable( + ref RowBuffer row, + ref RowCursor scope, + TypeArgument itemType, + out TValue item, + out RowCursor nullableScope) + where TValue : class + { + Result r = NullableUnitTests.ReadNullableImpl(ref row, ref scope, itemType, out item, out nullableScope); + return (r == Result.NotFound) ? Result.Success : r; + } + + private static Result ReadNullableImpl( + ref RowBuffer row, + ref RowCursor scope, + TypeArgument itemType, + out TValue item, + out RowCursor nullableScope) + { + Result r = itemType.Type.TypeAs().ReadScope(ref row, ref scope, out nullableScope); + if (r != Result.Success) + { + item = default; + return r; + } + + if (nullableScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(LayoutNullable.HasValue(ref row, ref nullableScope)); + return itemType.TypeArgs[0].Type.TypeAs>().ReadSparse(ref row, ref nullableScope, out item); + } + + ResultAssert.NotFound(LayoutNullable.HasValue(ref row, ref nullableScope)); + item = default; + return Result.NotFound; + } + + private void WriteNullables(ref RowBuffer row, ref RowCursor root, Nullables value) + { + LayoutColumn c; + + if (value.NullBool != null) + { + Assert.IsTrue(this.layout.TryFind("nullbool", out c)); + root.Clone(out RowCursor outerScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref outerScope, c.TypeArgs, out outerScope)); + foreach (bool? item in value.NullBool) + { + ResultAssert.IsSuccess(NullableUnitTests.WriteNullable(ref row, ref outerScope, c.TypeArgs[0], item, out RowCursor innerScope)); + Assert.IsFalse(outerScope.MoveNext(ref row, ref innerScope)); + } + } + + if (value.NullArray != null) + { + Assert.IsTrue(this.layout.TryFind("nullarray", out c)); + root.Clone(out RowCursor outerScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref outerScope, c.TypeArgs, out outerScope)); + foreach (float? item in value.NullArray) + { + ResultAssert.IsSuccess(NullableUnitTests.WriteNullable(ref row, ref outerScope, c.TypeArgs[0], item, out RowCursor innerScope)); + Assert.IsFalse(outerScope.MoveNext(ref row, ref innerScope)); + } + } + + if (value.NullSet != null) + { + Assert.IsTrue(this.layout.TryFind("nullset", out c)); + root.Clone(out RowCursor outerScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref outerScope, c.TypeArgs, out outerScope)); + foreach (string item in value.NullSet) + { + RowCursor.CreateForAppend(ref row, out RowCursor temp).Find(ref row, string.Empty); + ResultAssert.IsSuccess(NullableUnitTests.WriteNullable(ref row, ref temp, c.TypeArgs[0], item, out RowCursor _)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref outerScope, ref temp)); + } + } + + if (value.NullTuple != null) + { + Assert.IsTrue(this.layout.TryFind("nulltuple", out c)); + root.Clone(out RowCursor outerScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref outerScope, c.TypeArgs, out outerScope)); + foreach ((int? item1, long? item2) in value.NullTuple) + { + TypeArgument tupleType = c.TypeArgs[0]; + ResultAssert.IsSuccess( + tupleType.TypeAs().WriteScope(ref row, ref outerScope, tupleType.TypeArgs, out RowCursor tupleScope)); + + ResultAssert.IsSuccess( + NullableUnitTests.WriteNullable(ref row, ref tupleScope, tupleType.TypeArgs[0], item1, out RowCursor nullableScope)); + + Assert.IsTrue(tupleScope.MoveNext(ref row, ref nullableScope)); + ResultAssert.IsSuccess(NullableUnitTests.WriteNullable(ref row, ref tupleScope, tupleType.TypeArgs[1], item2, out nullableScope)); + Assert.IsFalse(tupleScope.MoveNext(ref row, ref nullableScope)); + + Assert.IsFalse(outerScope.MoveNext(ref row, ref tupleScope)); + } + } + + if (value.NullMap != null) + { + Assert.IsTrue(this.layout.TryFind("nullmap", out c)); + root.Clone(out RowCursor outerScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref outerScope, c.TypeArgs, out outerScope)); + foreach ((Guid key, byte? itemValue) in value.NullMap) + { + TypeArgument tupleType = c.TypeAs().FieldType(ref outerScope); + RowCursor.CreateForAppend(ref row, out RowCursor temp).Find(ref row, string.Empty); + ResultAssert.IsSuccess( + tupleType.TypeAs().WriteScope(ref row, ref temp, tupleType.TypeArgs, out RowCursor tupleScope)); + + Guid? itemKey = key.Equals(Guid.Empty) ? (Guid?)null : key; + ResultAssert.IsSuccess( + NullableUnitTests.WriteNullable(ref row, ref tupleScope, tupleType.TypeArgs[0], itemKey, out RowCursor nullableScope)); + + Assert.IsTrue(tupleScope.MoveNext(ref row, ref nullableScope)); + ResultAssert.IsSuccess( + NullableUnitTests.WriteNullable(ref row, ref tupleScope, tupleType.TypeArgs[1], itemValue, out nullableScope)); + + Assert.IsFalse(tupleScope.MoveNext(ref row, ref nullableScope)); + + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref outerScope, ref temp)); + } + } + } + + private Nullables ReadNullables(ref RowBuffer row, ref RowCursor root) + { + Nullables value = new Nullables(); + + Assert.IsTrue(this.layout.TryFind("nullbool", out LayoutColumn c)); + root.Clone(out RowCursor scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref scope, out scope) == Result.Success) + { + value.NullBool = new List(); + RowCursor nullableScope = default; + while (scope.MoveNext(ref row, ref nullableScope)) + { + ResultAssert.IsSuccess(NullableUnitTests.ReadNullable(ref row, ref scope, c.TypeArgs[0], out bool? item, out nullableScope)); + value.NullBool.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("nullarray", out c)); + root.Clone(out scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref scope, out scope) == Result.Success) + { + value.NullArray = new List(); + RowCursor nullableScope = default; + while (scope.MoveNext(ref row, ref nullableScope)) + { + ResultAssert.IsSuccess(NullableUnitTests.ReadNullable(ref row, ref scope, c.TypeArgs[0], out float? item, out nullableScope)); + value.NullArray.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("nullset", out c)); + root.Clone(out scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref scope, out scope) == Result.Success) + { + value.NullSet = new List(); + RowCursor nullableScope = default; + while (scope.MoveNext(ref row, ref nullableScope)) + { + ResultAssert.IsSuccess(NullableUnitTests.ReadNullable(ref row, ref scope, c.TypeArgs[0], out string item, out nullableScope)); + value.NullSet.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("nulltuple", out c)); + root.Clone(out scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref scope, out scope) == Result.Success) + { + value.NullTuple = new List<(int?, long?)>(); + RowCursor tupleScope = default; + TypeArgument tupleType = c.TypeArgs[0]; + while (scope.MoveNext(ref row, ref tupleScope)) + { + ResultAssert.IsSuccess(tupleType.TypeAs().ReadScope(ref row, ref scope, out tupleScope)); + + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + NullableUnitTests.ReadNullable(ref row, ref tupleScope, tupleType.TypeArgs[0], out int? item1, out RowCursor nullableScope)); + Assert.IsTrue(tupleScope.MoveNext(ref row, ref nullableScope)); + ResultAssert.IsSuccess( + NullableUnitTests.ReadNullable(ref row, ref tupleScope, tupleType.TypeArgs[1], out long? item2, out nullableScope)); + + Assert.IsFalse(tupleScope.MoveNext(ref row, ref nullableScope)); + value.NullTuple.Add((item1, item2)); + } + } + + Assert.IsTrue(this.layout.TryFind("nullmap", out c)); + root.Clone(out scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref scope, out scope) == Result.Success) + { + value.NullMap = new Dictionary(); + RowCursor tupleScope = default; + TypeArgument tupleType = c.TypeAs().FieldType(ref scope); + while (scope.MoveNext(ref row, ref tupleScope)) + { + ResultAssert.IsSuccess(tupleType.TypeAs().ReadScope(ref row, ref scope, out tupleScope)); + + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + NullableUnitTests.ReadNullable( + ref row, + ref tupleScope, + tupleType.TypeArgs[0], + out Guid? itemKey, + out RowCursor nullableScope)); + + Assert.IsTrue(tupleScope.MoveNext(ref row, ref nullableScope)); + ResultAssert.IsSuccess( + NullableUnitTests.ReadNullable(ref row, ref tupleScope, tupleType.TypeArgs[1], out byte? itemValue, out nullableScope)); + + Assert.IsFalse(tupleScope.MoveNext(ref row, ref nullableScope)); + value.NullMap.Add(itemKey ?? Guid.Empty, itemValue); + } + } + + return value; + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class Nullables + { + public List NullBool; + public List NullSet; + public List NullArray; + public List<(int?, long?)> NullTuple; + public Dictionary NullMap; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Nullables nullables && this.Equals(nullables); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = 0; + hashCode = (hashCode * 397) ^ (this.NullBool?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.NullSet?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.NullArray?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.NullTuple?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.NullMap?.GetHashCode() ?? 0); + return hashCode; + } + } + + private static bool MapEquals(Dictionary left, Dictionary right) + { + if (left.Count != right.Count) + { + return false; + } + + foreach (KeyValuePair item in left) + { + if (!right.TryGetValue(item.Key, out TValue value)) + { + return false; + } + + if (!item.Value.Equals(value)) + { + return false; + } + } + + return true; + } + + private bool Equals(Nullables other) + { + return (object.ReferenceEquals(this.NullBool, other.NullBool) || + ((this.NullBool != null) && (other.NullBool != null) && this.NullBool.SequenceEqual(other.NullBool))) && + (object.ReferenceEquals(this.NullSet, other.NullSet) || + ((this.NullSet != null) && (other.NullSet != null) && this.NullSet.SequenceEqual(other.NullSet))) && + (object.ReferenceEquals(this.NullArray, other.NullArray) || + ((this.NullArray != null) && (other.NullArray != null) && this.NullArray.SequenceEqual(other.NullArray))) && + (object.ReferenceEquals(this.NullTuple, other.NullTuple) || + ((this.NullTuple != null) && (other.NullTuple != null) && this.NullTuple.SequenceEqual(other.NullTuple))) && + (object.ReferenceEquals(this.NullMap, other.NullMap) || + ((this.NullMap != null) && (other.NullMap != null) && Nullables.MapEquals(this.NullMap, other.NullMap))); + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/PermuteExtensions.cs b/dotnet/src/HybridRow.Tests.Unit/PermuteExtensions.cs new file mode 100644 index 0000000..e1fd6ae --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/PermuteExtensions.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System.Collections.Generic; + using System.Linq; + + /// + /// Extension methods for computing permutations of . + /// + public static class PermuteExtensions + { + /// Generate all permutations of a given enumerable. + public static IEnumerable> Permute(this IEnumerable list) + { + int start = 0; + foreach (T element in list) + { + int index = start; + T[] first = { element }; + IEnumerable rest = list.Where((s, i) => i != index); + if (!rest.Any()) + { + yield return first; + } + + foreach (IEnumerable sub in rest.Permute()) + { + yield return first.Concat(sub); + } + + start++; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/Properties/AssemblyInfo.cs b/dotnet/src/HybridRow.Tests.Unit/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..2a8e8ee --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("DC93CAA3-9732-46D4-ACBF-D69EFC3F6511")] diff --git a/dotnet/src/HybridRow.Tests.Unit/RandomGeneratorUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/RandomGeneratorUnitTests.cs new file mode 100644 index 0000000..c78209e --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/RandomGeneratorUnitTests.cs @@ -0,0 +1,66 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class RandomGeneratorUnitTests + { + [TestMethod] + [Owner("jthunter")] + public void RangeTest() + { + int seed = 42; + RandomGenerator rand = new RandomGenerator(new Random(seed)); + ulong l1 = rand.NextUInt64(); + ulong l2 = rand.NextUInt64(); + Assert.AreNotEqual(l1, l2); + + Console.WriteLine("Check full range of min/max for ushort."); + for (int min = 0; min <= ushort.MaxValue; min++) + { + ushort i1 = rand.NextUInt16((ushort)min, ushort.MaxValue); + Assert.IsTrue(i1 >= min); + } + + Console.WriteLine("Check ushort range of min/max for uint."); + for (uint min = 0; min <= (uint)ushort.MaxValue; min++) + { + uint i1 = rand.NextUInt32(min, (uint)ushort.MaxValue); + Assert.IsTrue(i1 >= min); + Assert.IsTrue(i1 <= ushort.MaxValue); + } + + bool seenMax = false; + bool seenMin = false; + const ushort maxUShortRange = 10; + Console.WriteLine("Check inclusivity for ushort."); + while (!(seenMax && seenMin)) + { + ushort i1 = rand.NextUInt16(ushort.MinValue, maxUShortRange); + seenMin = seenMin || i1 == ushort.MinValue; + seenMax = seenMax || i1 == maxUShortRange; + Assert.IsTrue(i1 <= maxUShortRange); + } + + seenMax = false; + seenMin = false; + Console.WriteLine("Check inclusivity for short."); + const short minShortRange = -10; + const short maxShortRange = 10; + while (!(seenMax && seenMin)) + { + short i1 = rand.NextInt16(minShortRange, maxShortRange); + seenMin = seenMin || i1 == -10; + seenMax = seenMax || i1 == 10; + Assert.IsTrue(i1 >= minShortRange); + Assert.IsTrue(i1 <= maxShortRange); + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/RecordIOUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/RecordIOUnitTests.cs new file mode 100644 index 0000000..c6b3a74 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/RecordIOUnitTests.cs @@ -0,0 +1,198 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(RecordIOUnitTests.SchemaFile, "TestData")] + public class RecordIOUnitTests + { + private const string SchemaFile = @"TestData\CustomerSchema.json"; + private const int InitialRowSize = 0; + + private Namespace ns; + private LayoutResolver resolver; + private Layout addressLayout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(RecordIOUnitTests.SchemaFile); + this.ns = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.ns); + this.addressLayout = this.resolver.Resolve(this.ns.Schemas.Find(x => x.Name == "Address").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void LoadSchema() + { + LayoutResolver systemResolver = SystemSchema.LayoutResolver; + Layout segmentLayout = systemResolver.Resolve(SystemSchema.SegmentSchemaId); + Assert.AreEqual(segmentLayout.Name, "Segment"); + Assert.AreEqual(segmentLayout.SchemaId, SystemSchema.SegmentSchemaId); + + Layout recordLayout = systemResolver.Resolve(SystemSchema.RecordSchemaId); + Assert.AreEqual(recordLayout.Name, "Record"); + Assert.AreEqual(recordLayout.SchemaId, SystemSchema.RecordSchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public async Task RoundTripAsync() + { + Address[] addresses = + { + new Address + { + Street = "300B Chocolate Hole", + City = "Great Cruz Bay", + State = "VI", + PostalCode = new PostalCode + { + Zip = 00830, + Plus4 = 0001, + }, + }, + new Address + { + Street = "1 Microsoft Way", + City = "Redmond", + State = "WA", + PostalCode = new PostalCode + { + Zip = 98052, + }, + }, + }; + + string sampleComment = "hello there"; + string sampleSDL = "some SDL"; + + using (Stream stm = new MemoryStream()) + { + // Create a reusable, resizable buffer. + MemorySpanResizer resizer = new MemorySpanResizer(RecordIOUnitTests.InitialRowSize); + + // Write a RecordIO stream. + Result r = await stm.WriteRecordIOAsync( + new Segment(sampleComment, sampleSDL), + (long index, out ReadOnlyMemory body) => + { + body = default; + if (index >= addresses.Length) + { + return Result.Success; + } + + return this.WriteAddress(resizer, addresses[index], out body); + }); + + // Read a RecordIO stream. + List
addressesRead = new List
(); + stm.Position = 0; + resizer = new MemorySpanResizer(1); + r = await stm.ReadRecordIOAsync( + record => + { + Assert.IsFalse(record.IsEmpty); + + r = this.ReadAddress(record, out Address obj); + ResultAssert.IsSuccess(r); + addressesRead.Add(obj); + return Result.Success; + }, + segment => + { + Assert.IsFalse(segment.IsEmpty); + + r = this.ReadSegment(segment, out Segment obj); + ResultAssert.IsSuccess(r); + Assert.AreEqual(obj.Comment, sampleComment); + Assert.AreEqual(obj.SDL, sampleSDL); + return Result.Success; + }, + resizer); + + ResultAssert.IsSuccess(r); + + // Check that the values all round-tripped. + Assert.AreEqual(addresses.Length, addressesRead.Count); + for (int i = 0; i < addresses.Length; i++) + { + Assert.AreEqual(addresses[i], addressesRead[i]); + } + } + } + + private Result WriteAddress(MemorySpanResizer resizer, Address obj, out ReadOnlyMemory buffer) + { + RowBuffer row = new RowBuffer(RecordIOUnitTests.InitialRowSize, resizer); + row.InitLayout(HybridRowVersion.V1, this.addressLayout, this.resolver); + Result r = RowWriter.WriteBuffer(ref row, obj, AddressSerializer.Write); + if (r != Result.Success) + { + buffer = default; + return r; + } + + buffer = resizer.Memory.Slice(0, row.Length); + return Result.Success; + } + + private Result ReadAddress(Memory buffer, out Address obj) + { + RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, this.resolver); + RowReader reader = new RowReader(ref row); + + // Use the reader to dump to the screen. + Result r = DiagnosticConverter.ReaderToString(ref reader, out string str); + if (r != Result.Success) + { + obj = default; + return r; + } + + Console.WriteLine(str); + + // Reset the reader and materialize the object. + reader = new RowReader(ref row); + return AddressSerializer.Read(ref reader, out obj); + } + + private Result ReadSegment(Memory buffer, out Segment obj) + { + RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, SystemSchema.LayoutResolver); + RowReader reader = new RowReader(ref row); + + // Use the reader to dump to the screen. + Result r = DiagnosticConverter.ReaderToString(ref reader, out string str); + if (r != Result.Success) + { + obj = default; + return r; + } + + Console.WriteLine(str); + + // Reset the reader and materialize the object. + reader = new RowReader(ref row); + return SegmentSerializer.Read(ref reader, out obj); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/ResultAssert.cs b/dotnet/src/HybridRow.Tests.Unit/ResultAssert.cs new file mode 100644 index 0000000..ccc436f --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/ResultAssert.cs @@ -0,0 +1,101 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + + internal static class ResultAssert + { + public static void IsSuccess(Result actual) + { + Assert.AreEqual(Result.Success, actual); + } + + public static void IsSuccess(Result actual, string message) + { + Assert.AreEqual(Result.Success, actual, message); + } + + public static void IsSuccess(Result actual, string message, params object[] parameters) + { + Assert.AreEqual(Result.Success, actual, message, parameters); + } + + public static void NotFound(Result actual) + { + Assert.AreEqual(Result.NotFound, actual); + } + + public static void NotFound(Result actual, string message) + { + Assert.AreEqual(Result.NotFound, actual, message); + } + + public static void NotFound(Result actual, string message, params object[] parameters) + { + Assert.AreEqual(Result.NotFound, actual, message, parameters); + } + + public static void Exists(Result actual) + { + Assert.AreEqual(Result.Exists, actual); + } + + public static void Exists(Result actual, string message) + { + Assert.AreEqual(Result.Exists, actual, message); + } + + public static void Exists(Result actual, string message, params object[] parameters) + { + Assert.AreEqual(Result.Exists, actual, message, parameters); + } + + public static void TypeMismatch(Result actual) + { + Assert.AreEqual(Result.TypeMismatch, actual); + } + + public static void TypeMismatch(Result actual, string message) + { + Assert.AreEqual(Result.TypeMismatch, actual, message); + } + + public static void TypeMismatch(Result actual, string message, params object[] parameters) + { + Assert.AreEqual(Result.TypeMismatch, actual, message, parameters); + } + + public static void InsufficientPermissions(Result actual) + { + Assert.AreEqual(Result.InsufficientPermissions, actual); + } + + public static void InsufficientPermissions(Result actual, string message) + { + Assert.AreEqual(Result.InsufficientPermissions, actual, message); + } + + public static void InsufficientPermissions(Result actual, string message, params object[] parameters) + { + Assert.AreEqual(Result.InsufficientPermissions, actual, message, parameters); + } + + public static void TypeConstraint(Result actual) + { + Assert.AreEqual(Result.TypeConstraint, actual); + } + + public static void TypeConstraint(Result actual, string message) + { + Assert.AreEqual(Result.TypeConstraint, actual, message); + } + + public static void TypeConstraint(Result actual, string message, params object[] parameters) + { + Assert.AreEqual(Result.TypeConstraint, actual, message, parameters); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/RowBufferUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/RowBufferUnitTests.cs new file mode 100644 index 0000000..619f543 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/RowBufferUnitTests.cs @@ -0,0 +1,60 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System.Diagnostics.CodeAnalysis; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class RowBufferUnitTests + { + [TestMethod] + [Owner("jthunter")] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1139:UseLiteralsSuffixNotationInsteadOfCasting", Justification = "Explicit")] + public void VarIntTest() + { + // Brute force test all signed 16-bit values. + for (int i = short.MinValue; i <= short.MaxValue; i++) + { + short s = (short)i; + this.RoundTripVarInt(s); + } + + // Test boundary conditions for larger values. + this.RoundTripVarInt(0); + this.RoundTripVarInt(int.MinValue); + this.RoundTripVarInt(unchecked((int)0x80000000ul)); + this.RoundTripVarInt(unchecked((int)0x7FFFFFFFul)); + this.RoundTripVarInt(int.MaxValue); + this.RoundTripVarInt(long.MinValue); + this.RoundTripVarInt(unchecked((long)0x8000000000000000ul)); + this.RoundTripVarInt(unchecked((long)0x7FFFFFFFFFFFFFFFul)); + this.RoundTripVarInt(long.MaxValue); + } + + private void RoundTripVarInt(short s) + { + ulong encoded = RowBuffer.RotateSignToLsb(s); + long decoded = RowBuffer.RotateSignToMsb(encoded); + short t = unchecked((short)decoded); + Assert.AreEqual(s, t, "Value: {0}", s); + } + + private void RoundTripVarInt(int s) + { + ulong encoded = RowBuffer.RotateSignToLsb(s); + long decoded = RowBuffer.RotateSignToMsb(encoded); + int t = unchecked((int)decoded); + Assert.AreEqual(s, t, "Value: {0}", s); + } + + private void RoundTripVarInt(long s) + { + ulong encoded = RowBuffer.RotateSignToLsb(s); + long decoded = RowBuffer.RotateSignToMsb(encoded); + Assert.AreEqual(s, decoded, "Value: {0}", s); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/RowOperationDispatcher.cs b/dotnet/src/HybridRow.Tests.Unit/RowOperationDispatcher.cs new file mode 100644 index 0000000..19ea837 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/RowOperationDispatcher.cs @@ -0,0 +1,890 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1402 // FileMayOnlyContainASingleType +#pragma warning disable SA1201 // OrderingRules +#pragma warning disable SA1401 // Public Fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections; + using System.IO; + using System.Reflection; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + internal ref struct RowOperationDispatcher + { + public readonly LayoutResolver Resolver; + public RowBuffer Row; + + private const int InitialRowSize = 2 * 1024 * 1024; + private readonly IDispatcher dispatcher; + + private RowOperationDispatcher(IDispatcher dispatcher, Layout layout, LayoutResolver resolver) + { + this.dispatcher = dispatcher; + this.Row = new RowBuffer(RowOperationDispatcher.InitialRowSize); + this.Resolver = resolver; + this.Row.InitLayout(HybridRowVersion.V1, layout, this.Resolver); + } + + private RowOperationDispatcher(IDispatcher dispatcher, LayoutResolver resolver, string expected) + { + this.dispatcher = dispatcher; + this.Row = new RowBuffer(RowOperationDispatcher.InitialRowSize); + this.Resolver = resolver; + byte[] bytes = ByteConverter.ToBytes(expected); + this.Row.ReadFrom(bytes, HybridRowVersion.V1, this.Resolver); + } + + public static RowOperationDispatcher Create(Layout layout, LayoutResolver resolver) + where TDispatcher : struct, IDispatcher + { + return new RowOperationDispatcher(default(TDispatcher), layout, resolver); + } + + public static RowOperationDispatcher ReadFrom(LayoutResolver resolver, string expected) + where TDispatcher : struct, IDispatcher + { + return new RowOperationDispatcher(default(TDispatcher), resolver, expected); + } + + public string RowToHex() + { + using (MemoryStream stm = new MemoryStream()) + { + this.Row.WriteTo(stm); + ReadOnlyMemory bytes = stm.GetBuffer().AsMemory(0, (int)stm.Position); + return ByteConverter.ToHex(bytes.Span); + } + } + + public RowReader GetReader() + { + return new RowReader(ref this.Row); + } + + public void LayoutCodeSwitch( + string path = null, + LayoutType type = null, + TypeArgumentList typeArgs = default, + object value = null) + { + RowCursor root = RowCursor.Create(ref this.Row); + this.LayoutCodeSwitch(ref root, path, type, typeArgs, value); + } + + public void LayoutCodeSwitch( + ref RowCursor scope, + string path = null, + LayoutType type = null, + TypeArgumentList typeArgs = default, + object value = null) + { + LayoutColumn col = null; + if (type == null) + { + Assert.IsNotNull(path); + Assert.IsTrue(scope.Layout.TryFind(path, out col)); + Assert.IsNotNull(col); + type = col.Type; + typeArgs = col.TypeArgs; + } + + if ((path != null) && (col == null || col.Storage == StorageKind.Sparse)) + { + scope.Find(ref this.Row, path); + } + + switch (type.LayoutCode) + { + case LayoutCode.Null: + this.dispatcher.Dispatch(ref this, ref scope, col, type, NullValue.Default); + break; + case LayoutCode.Boolean: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (bool?)value ?? default); + + break; + case LayoutCode.Int8: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (sbyte?)value ?? default); + + break; + case LayoutCode.Int16: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (short?)value ?? default); + + break; + case LayoutCode.Int32: + this.dispatcher.Dispatch(ref this, ref scope, col, type, (int?)value ?? default); + break; + case LayoutCode.Int64: + this.dispatcher.Dispatch(ref this, ref scope, col, type, (long?)value ?? default); + break; + case LayoutCode.UInt8: + this.dispatcher.Dispatch(ref this, ref scope, col, type, (byte?)value ?? default); + break; + case LayoutCode.UInt16: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (ushort?)value ?? default); + + break; + case LayoutCode.UInt32: + this.dispatcher.Dispatch(ref this, ref scope, col, type, (uint?)value ?? default); + break; + case LayoutCode.UInt64: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (ulong?)value ?? default); + + break; + case LayoutCode.VarInt: + this.dispatcher.Dispatch(ref this, ref scope, col, type, (long?)value ?? default); + break; + case LayoutCode.VarUInt: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (ulong?)value ?? default); + + break; + case LayoutCode.Float32: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (float?)value ?? default); + + break; + case LayoutCode.Float64: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (double?)value ?? default); + + break; + case LayoutCode.Float128: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (Float128?)value ?? default); + + break; + case LayoutCode.Decimal: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (decimal?)value ?? default); + + break; + case LayoutCode.DateTime: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (DateTime?)value ?? default); + + break; + case LayoutCode.UnixDateTime: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (UnixDateTime?)value ?? default); + + break; + case LayoutCode.Guid: + this.dispatcher.Dispatch(ref this, ref scope, col, type, (Guid?)value ?? default); + break; + case LayoutCode.MongoDbObjectId: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (MongoDbObjectId?)value ?? default); + + break; + case LayoutCode.Utf8: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (string)value); + + break; + case LayoutCode.Binary: + this.dispatcher.Dispatch( + ref this, + ref scope, + col, + type, + (byte[])value); + + break; + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + this.dispatcher.DispatchObject(ref this, ref scope); + break; + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + this.dispatcher.DispatchArray(ref this, ref scope, type, typeArgs, value); + break; + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + this.dispatcher.DispatchSet(ref this, ref scope, type, typeArgs, value); + break; + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + this.dispatcher.DispatchMap(ref this, ref scope, type, typeArgs, value); + break; + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + this.dispatcher.DispatchTuple(ref this, ref scope, type, typeArgs, value); + break; + case LayoutCode.NullableScope: + this.dispatcher.DispatchNullable(ref this, ref scope, type, typeArgs, value); + break; + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + this.dispatcher.DispatchUDT(ref this, ref scope, type, typeArgs, value); + break; + default: + Contract.Assert(false, $"Unknown type will be ignored: {type.LayoutCode}"); + break; + } + } + } + + internal interface IDispatcher + { + void Dispatch( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutColumn col, + LayoutType t, + TValue value = default) + where TLayout : LayoutType; + + void DispatchObject(ref RowOperationDispatcher dispatcher, ref RowCursor scope); + + void DispatchArray( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value); + + void DispatchTuple( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value); + + void DispatchNullable( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value); + + void DispatchSet( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value); + + void DispatchMap( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value); + + void DispatchUDT( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType type, + TypeArgumentList typeArgs, + object value); + } + + internal interface IDispatchable + { + void Dispatch(ref RowOperationDispatcher dispatcher, ref RowCursor scope); + } + + internal struct WriteRowDispatcher : IDispatcher + { + public void Dispatch( + ref RowOperationDispatcher dispatcher, + ref RowCursor field, + LayoutColumn col, + LayoutType t, + TValue value = default) + where TLayout : LayoutType + { + switch (col?.Storage) + { + case StorageKind.Fixed: + ResultAssert.IsSuccess(t.TypeAs().WriteFixed(ref dispatcher.Row, ref field, col, value)); + break; + case StorageKind.Variable: + ResultAssert.IsSuccess(t.TypeAs().WriteVariable(ref dispatcher.Row, ref field, col, value)); + break; + default: + ResultAssert.IsSuccess(t.TypeAs().WriteSparse(ref dispatcher.Row, ref field, value)); + break; + } + } + + public void DispatchObject(ref RowOperationDispatcher dispatcher, ref RowCursor scope) + { + } + + public void DispatchArray( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess( + t.TypeAs().WriteScope(ref dispatcher.Row, ref scope, typeArgs, out RowCursor arrayScope)); + + IList items = (IList)value; + foreach (object item in items) + { + dispatcher.LayoutCodeSwitch(ref arrayScope, null, typeArgs[0].Type, typeArgs[0].TypeArgs, item); + arrayScope.MoveNext(ref dispatcher.Row); + } + } + + public void DispatchTuple( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count >= 2); + + ResultAssert.IsSuccess( + t.TypeAs().WriteScope(ref dispatcher.Row, ref scope, typeArgs, out RowCursor tupleScope)); + + for (int i = 0; i < typeArgs.Count; i++) + { + PropertyInfo valueAccessor = value.GetType().GetProperty($"Item{i + 1}"); + dispatcher.LayoutCodeSwitch( + ref tupleScope, + null, + typeArgs[i].Type, + typeArgs[i].TypeArgs, + valueAccessor.GetValue(value)); + tupleScope.MoveNext(ref dispatcher.Row); + } + } + + public void DispatchNullable( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess( + t.TypeAs() + .WriteScope(ref dispatcher.Row, ref scope, typeArgs, value != null, out RowCursor nullableScope)); + + if (value != null) + { + dispatcher.LayoutCodeSwitch(ref nullableScope, null, typeArgs[0].Type, typeArgs[0].TypeArgs, value); + } + } + + public void DispatchSet( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess(t.TypeAs().WriteScope(ref dispatcher.Row, ref scope, typeArgs, out RowCursor setScope)); + IList items = (IList)value; + foreach (object item in items) + { + string elmPath = Guid.NewGuid().ToString(); + RowCursor.CreateForAppend(ref dispatcher.Row, out RowCursor tempCursor); + dispatcher.LayoutCodeSwitch(ref tempCursor, elmPath, typeArgs[0].Type, typeArgs[0].TypeArgs, item); + + // Move item into the set. + ResultAssert.IsSuccess(t.TypeAs().MoveField(ref dispatcher.Row, ref setScope, ref tempCursor)); + } + } + + public void DispatchMap( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 2); + + ResultAssert.IsSuccess(t.TypeAs().WriteScope(ref dispatcher.Row, ref scope, typeArgs, out RowCursor mapScope)); + TypeArgument fieldType = t.TypeAs().FieldType(ref mapScope); + IList pairs = (IList)value; + foreach (object pair in pairs) + { + string elmPath = Guid.NewGuid().ToString(); + RowCursor.CreateForAppend(ref dispatcher.Row, out RowCursor tempCursor); + dispatcher.LayoutCodeSwitch(ref tempCursor, elmPath, fieldType.Type, fieldType.TypeArgs, pair); + + // Move item into the map. + ResultAssert.IsSuccess(t.TypeAs().MoveField(ref dispatcher.Row, ref mapScope, ref tempCursor)); + } + } + + public void DispatchUDT( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + ResultAssert.IsSuccess(t.TypeAs().WriteScope(ref dispatcher.Row, ref scope, typeArgs, out RowCursor udtScope)); + IDispatchable valueDispatcher = value as IDispatchable; + Assert.IsNotNull(valueDispatcher); + valueDispatcher.Dispatch(ref dispatcher, ref udtScope); + } + } + + internal struct ReadRowDispatcher : IDispatcher + { + public void Dispatch( + ref RowOperationDispatcher dispatcher, + ref RowCursor root, + LayoutColumn col, + LayoutType t, + TValue expected = default) + where TLayout : LayoutType + { + TValue value; + switch (col?.Storage) + { + case StorageKind.Fixed: + ResultAssert.IsSuccess(t.TypeAs().ReadFixed(ref dispatcher.Row, ref root, col, out value)); + break; + case StorageKind.Variable: + ResultAssert.IsSuccess(t.TypeAs().ReadVariable(ref dispatcher.Row, ref root, col, out value)); + break; + default: + ResultAssert.IsSuccess(t.TypeAs().ReadSparse(ref dispatcher.Row, ref root, out value)); + break; + } + + if (typeof(TValue).IsArray) + { + CollectionAssert.AreEqual((ICollection)expected, (ICollection)value); + } + else + { + Assert.AreEqual(expected, value); + } + } + + public void DispatchObject(ref RowOperationDispatcher dispatcher, ref RowCursor scope) + { + } + + public void DispatchArray( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess( + t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor arrayScope)); + + int i = 0; + IList items = (IList)value; + while (arrayScope.MoveNext(ref dispatcher.Row)) + { + dispatcher.LayoutCodeSwitch( + ref arrayScope, + null, + typeArgs[0].Type, + typeArgs[0].TypeArgs, + items[i++]); + } + } + + public void DispatchTuple( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count >= 2); + + ResultAssert.IsSuccess( + t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor tupleScope)); + + for (int i = 0; i < typeArgs.Count; i++) + { + tupleScope.MoveNext(ref dispatcher.Row); + PropertyInfo valueAccessor = value.GetType().GetProperty($"Item{i + 1}"); + dispatcher.LayoutCodeSwitch( + ref tupleScope, + null, + typeArgs[i].Type, + typeArgs[i].TypeArgs, + valueAccessor.GetValue(value)); + } + } + + public void DispatchNullable( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess( + t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor nullableScope)); + + if (value != null) + { + ResultAssert.IsSuccess(LayoutNullable.HasValue(ref dispatcher.Row, ref nullableScope)); + nullableScope.MoveNext(ref dispatcher.Row); + dispatcher.LayoutCodeSwitch(ref nullableScope, null, typeArgs[0].Type, typeArgs[0].TypeArgs, value); + } + else + { + ResultAssert.NotFound(LayoutNullable.HasValue(ref dispatcher.Row, ref nullableScope)); + } + } + + public void DispatchSet( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess(t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor setScope)); + int i = 0; + IList items = (IList)value; + while (setScope.MoveNext(ref dispatcher.Row)) + { + dispatcher.LayoutCodeSwitch( + ref setScope, + null, + typeArgs[0].Type, + typeArgs[0].TypeArgs, + items[i++]); + } + } + + public void DispatchMap( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 2); + + ResultAssert.IsSuccess(t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor mapScope)); + int i = 0; + IList items = (IList)value; + while (mapScope.MoveNext(ref dispatcher.Row)) + { + dispatcher.LayoutCodeSwitch( + ref mapScope, + null, + LayoutType.TypedTuple, + typeArgs, + items[i++]); + } + } + + public void DispatchUDT( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + ResultAssert.IsSuccess(t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor udtScope)); + IDispatchable valueDispatcher = value as IDispatchable; + Assert.IsNotNull(valueDispatcher); + valueDispatcher.Dispatch(ref dispatcher, ref udtScope); + } + } + + internal struct DeleteRowDispatcher : IDispatcher + { + public void Dispatch( + ref RowOperationDispatcher dispatcher, + ref RowCursor root, + LayoutColumn col, + LayoutType t, + TValue value = default) + where TLayout : LayoutType + { + ResultAssert.IsSuccess(t.TypeAs().DeleteSparse(ref dispatcher.Row, ref root)); + } + + public void DispatchObject(ref RowOperationDispatcher dispatcher, ref RowCursor scope) + { + } + + public void DispatchArray( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess( + t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor arrayScope)); + + if (!arrayScope.Immutable) + { + IList items = (IList)value; + foreach (object item in items) + { + Assert.IsTrue(arrayScope.MoveNext(ref dispatcher.Row)); + dispatcher.LayoutCodeSwitch(ref arrayScope, null, typeArgs[0].Type, typeArgs[0].TypeArgs, item); + } + } + + ResultAssert.IsSuccess(t.TypeAs().DeleteScope(ref dispatcher.Row, ref scope)); + } + + public void DispatchTuple( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count >= 2); + ResultAssert.IsSuccess(t.TypeAs().DeleteScope(ref dispatcher.Row, ref scope)); + } + + public void DispatchNullable( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + ResultAssert.IsSuccess(t.TypeAs().DeleteScope(ref dispatcher.Row, ref scope)); + } + + public void DispatchSet( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 1); + + ResultAssert.IsSuccess(t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor setScope)); + if (!setScope.Immutable) + { + IList items = (IList)value; + foreach (object item in items) + { + Assert.IsTrue(setScope.MoveNext(ref dispatcher.Row)); + dispatcher.LayoutCodeSwitch(ref setScope, null, typeArgs[0].Type, typeArgs[0].TypeArgs, item); + } + } + + ResultAssert.IsSuccess(t.TypeAs().DeleteScope(ref dispatcher.Row, ref scope)); + } + + public void DispatchMap( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + Contract.Requires(typeArgs.Count == 2); + + ResultAssert.IsSuccess(t.TypeAs().ReadScope(ref dispatcher.Row, ref scope, out RowCursor mapScope)); + if (!mapScope.Immutable) + { + IList items = (IList)value; + foreach (object item in items) + { + Assert.IsTrue(mapScope.MoveNext(ref dispatcher.Row)); + dispatcher.LayoutCodeSwitch(ref mapScope, null, LayoutType.TypedTuple, typeArgs, item); + } + } + + ResultAssert.IsSuccess(t.TypeAs().DeleteScope(ref dispatcher.Row, ref scope)); + } + + public void DispatchUDT( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + ResultAssert.IsSuccess(t.TypeAs().DeleteScope(ref dispatcher.Row, ref scope)); + } + } + + internal struct NullRowDispatcher : IDispatcher + { + public void Dispatch( + ref RowOperationDispatcher dispatcher, + ref RowCursor root, + LayoutColumn col, + LayoutType t, + TValue expected = default) + where TLayout : LayoutType + { + switch (col?.Storage) + { + case StorageKind.Fixed: + ResultAssert.NotFound(t.TypeAs().ReadFixed(ref dispatcher.Row, ref root, col, out TValue _)); + break; + case StorageKind.Variable: + ResultAssert.NotFound(t.TypeAs().ReadVariable(ref dispatcher.Row, ref root, col, out TValue _)); + break; + default: + ResultAssert.NotFound(t.TypeAs().ReadSparse(ref dispatcher.Row, ref root, out TValue _)); + break; + } + } + + public void DispatchObject(ref RowOperationDispatcher dispatcher, ref RowCursor scope) + { + } + + public void DispatchArray( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + } + + public void DispatchTuple( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + } + + public void DispatchNullable( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + } + + public void DispatchSet( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + } + + public void DispatchMap( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + } + + public void DispatchUDT( + ref RowOperationDispatcher dispatcher, + ref RowCursor scope, + LayoutType t, + TypeArgumentList typeArgs, + object value) + { + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/RowReaderUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/RowReaderUnitTests.cs new file mode 100644 index 0000000..d3cd6f7 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/RowReaderUnitTests.cs @@ -0,0 +1,353 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1118:ParameterMustNotSpanMultipleLines", Justification = "Test code.")] + [DeploymentItem(RowReaderUnitTests.SchemaFile, "TestData")] + public sealed class RowReaderUnitTests + { + private const string SchemaFile = @"TestData\ReaderSchema.json"; + private static readonly DateTime SampleDateTime = DateTime.Parse("2018-08-14 02:05:00.0000000"); + private static readonly Guid SampleGuid = Guid.Parse("{2A9C25B9-922E-4611-BB0A-244A9496503C}"); + private static readonly Float128 SampleFloat128 = new Float128(0, 42); + private static readonly UnixDateTime SampleUnixDateTime = new UnixDateTime(42); + private static readonly MongoDbObjectId SampleMongoDbObjectId = new MongoDbObjectId(0, 42); + + private Namespace schema; + private LayoutResolver resolver; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(RowReaderUnitTests.SchemaFile); + this.schema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.schema); + } + + [TestMethod] + [Owner("jthunter")] + public void ReadMixed() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Mixed").SchemaId); + Assert.IsNotNull(layout); + + RowOperationDispatcher d = RowOperationDispatcher.Create(layout, this.resolver); + d.LayoutCodeSwitch("null"); + d.LayoutCodeSwitch("bool", value: true); + d.LayoutCodeSwitch("int8", value: (sbyte)-86); + d.LayoutCodeSwitch("int16", value: (short)-21846); + d.LayoutCodeSwitch("int32", value: -1431655766); + d.LayoutCodeSwitch("int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("float128", value: RowReaderUnitTests.SampleFloat128); + d.LayoutCodeSwitch("decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("datetime", value: RowReaderUnitTests.SampleDateTime); + d.LayoutCodeSwitch("unixdatetime", value: RowReaderUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("guid", value: RowReaderUnitTests.SampleGuid); + d.LayoutCodeSwitch("mongodbobjectid", value: RowReaderUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("utf8", value: "abc"); + d.LayoutCodeSwitch("utf8_span", value: "abc"); + d.LayoutCodeSwitch("binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("binary_span", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("var_varint", value: -6148914691236517206L); + d.LayoutCodeSwitch("var_varuint", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("var_utf8", value: "abc"); + d.LayoutCodeSwitch("var_utf8_span", value: "abc"); + d.LayoutCodeSwitch("var_binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("var_binary_span", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("sparse_null"); + d.LayoutCodeSwitch("sparse_bool", value: true); + d.LayoutCodeSwitch("sparse_int8", value: (sbyte)-86); + d.LayoutCodeSwitch("sparse_int16", value: (short)-21846); + d.LayoutCodeSwitch("sparse_int32", value: -1431655766); + d.LayoutCodeSwitch("sparse_int64", value: -6148914691236517206L); + d.LayoutCodeSwitch("sparse_uint8", value: (byte)0xAA); + d.LayoutCodeSwitch("sparse_uint16", value: (ushort)0xAAAA); + d.LayoutCodeSwitch("sparse_uint32", value: 0xAAAAAAAA); + d.LayoutCodeSwitch("sparse_uint64", value: 0xAAAAAAAAAAAAAAAAL); + d.LayoutCodeSwitch("sparse_float32", value: 1.0F / 3.0F); + d.LayoutCodeSwitch("sparse_float64", value: 1.0 / 3.0); + d.LayoutCodeSwitch("sparse_float128", value: RowReaderUnitTests.SampleFloat128); + d.LayoutCodeSwitch("sparse_decimal", value: 1.0M / 3.0M); + d.LayoutCodeSwitch("sparse_datetime", value: RowReaderUnitTests.SampleDateTime); + d.LayoutCodeSwitch("sparse_unixdatetime", value: RowReaderUnitTests.SampleUnixDateTime); + d.LayoutCodeSwitch("sparse_guid", value: RowReaderUnitTests.SampleGuid); + d.LayoutCodeSwitch("sparse_mongodbobjectid", value: RowReaderUnitTests.SampleMongoDbObjectId); + d.LayoutCodeSwitch("sparse_utf8", value: "abc"); + d.LayoutCodeSwitch("sparse_utf8_span", value: "abc"); + d.LayoutCodeSwitch("sparse_binary", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("sparse_binary_span", value: new[] { (byte)0, (byte)1, (byte)2 }); + d.LayoutCodeSwitch("array_t", value: new sbyte[] { -86, -86, -86 }); + d.LayoutCodeSwitch("array_t>", value: new[] { new float[] { 1, 2, 3 }, new float[] { 1, 2, 3 } }); + d.LayoutCodeSwitch("array_t", value: new[] { "abc", "def", "hij" }); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(-6148914691236517206L, -6148914691236517206L)); + d.LayoutCodeSwitch("tuple>", value: Tuple.Create(NullValue.Default, Tuple.Create((sbyte)-86, (sbyte)-86))); + d.LayoutCodeSwitch("tuple", value: Tuple.Create(false, new Point(1, 2))); + d.LayoutCodeSwitch("nullable", value: Tuple.Create(default(int?), (long?)123L)); + d.LayoutCodeSwitch("tagged", value: Tuple.Create((byte)3, "hello")); + d.LayoutCodeSwitch("tagged", value: Tuple.Create((byte)5, true, "bye")); + d.LayoutCodeSwitch("set_t", value: new[] { "abc", "efg", "xzy" }); + d.LayoutCodeSwitch("set_t>", value: new[] { new sbyte[] { 1, 2, 3 }, new sbyte[] { 4, 5, 6 }, new sbyte[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t>", value: new[] { new[] { 1, 2, 3 }, new[] { 4, 5, 6 }, new[] { 7, 8, 9 } }); + d.LayoutCodeSwitch("set_t", value: new[] { new Point(1, 2), new Point(3, 4), new Point(5, 6) }); + d.LayoutCodeSwitch("map_t", value: new[] { Tuple.Create("Mark", "Luke"), Tuple.Create("Harrison", "Han") }); + d.LayoutCodeSwitch( + "map_t>", + value: new[] { Tuple.Create((sbyte)1, new sbyte[] { 1, 2, 3 }), Tuple.Create((sbyte)2, new sbyte[] { 4, 5, 6 }) }); + + d.LayoutCodeSwitch( + "map_t>", + value: new[] + { + Tuple.Create((short)1, new[] { Tuple.Create(1, 2), Tuple.Create(3, 4) }), + Tuple.Create((short)2, new[] { Tuple.Create(5, 6), Tuple.Create(7, 8) }), + }); + + d.LayoutCodeSwitch( + "map_t", + value: new[] + { + Tuple.Create(1.0, new Point(1, 2)), + Tuple.Create(2.0, new Point(3, 4)), + Tuple.Create(3.0, new Point(5, 6)), + }); + + RowReader reader = d.GetReader(); + Assert.AreEqual(reader.Length, d.Row.Length); + RowReaderUnitTests.PrintReader(ref reader, 0); + } + + [TestMethod] + [Owner("jthunter")] + public void ReadScopes() + { + MemorySpanResizer resizer = new MemorySpanResizer(0); + RowBuffer row = new RowBuffer(0, resizer); + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Mixed").SchemaId); + row.InitLayout(HybridRowVersion.V1, layout, this.resolver); + + ResultAssert.IsSuccess(RowWriter.WriteBuffer(ref row, 2, RowReaderUnitTests.WriteNestedDocument)); + RowReader rowReader = new RowReader(ref row); + + ResultAssert.IsSuccess(RowReaderUnitTests.ReadNestedDocumentDelegate(ref rowReader, 0)); + + rowReader = new RowReader(ref row); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadNestedDocumentNonDelegate(ref rowReader, 0)); + + rowReader = new RowReader(ref row); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadNestedDocumentNonDelegateWithSkipScope(ref rowReader, 0)); + + // SkipScope not okay after advancing parent + rowReader = new RowReader(ref row); + Assert.IsTrue(rowReader.Read()); + Assert.AreEqual(rowReader.Type.LayoutCode, LayoutCode.ObjectScope); + RowReader nestedScope = rowReader.ReadScope(); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadNestedDocumentDelegate(ref nestedScope, 0)); + Assert.IsTrue(rowReader.Read()); + Result result = rowReader.SkipScope(ref nestedScope); + Assert.AreNotEqual(Result.Success, result); + } + + internal static void PrintReader(ref RowReader reader, int indent) + { + string str; + ResultAssert.IsSuccess(DiagnosticConverter.ReaderToString(ref reader, out str)); + Console.WriteLine(str); + } + + private static Result WriteNestedDocument(ref RowWriter writer, TypeArgument typeArgument, int level) + { + TypeArgument tupleArgument = new TypeArgument( + LayoutType.Tuple, + new TypeArgumentList(new[] + { + new TypeArgument(LayoutType.Int32), + new TypeArgument(LayoutType.Int32), + new TypeArgument(LayoutType.Int32), + })); + + Result WriteTuple(ref RowWriter tupleWriter, TypeArgument tupleTypeArgument, int unused) + { + ResultAssert.IsSuccess(tupleWriter.WriteInt32(null, 1)); + ResultAssert.IsSuccess(tupleWriter.WriteInt32(null, 2)); + ResultAssert.IsSuccess(tupleWriter.WriteInt32(null, 3)); + return Result.Success; + } + + if (level == 0) + { + ResultAssert.IsSuccess(writer.WriteScope("x", tupleArgument, 0, WriteTuple)); + return Result.Success; + } + + ResultAssert.IsSuccess(writer.WriteScope("a", new TypeArgument(LayoutType.Object), level - 1, RowReaderUnitTests.WriteNestedDocument)); + ResultAssert.IsSuccess(writer.WriteScope("x", tupleArgument, 0, WriteTuple)); + ResultAssert.IsSuccess(writer.WriteScope("b", new TypeArgument(LayoutType.Object), level - 1, RowReaderUnitTests.WriteNestedDocument)); + ResultAssert.IsSuccess(writer.WriteScope("y", tupleArgument, 0, WriteTuple)); + ResultAssert.IsSuccess(writer.WriteScope("c", new TypeArgument(LayoutType.Object), level - 1, RowReaderUnitTests.WriteNestedDocument)); + + return Result.Success; + } + + private static Result ReadNestedDocumentDelegate(ref RowReader reader, int context) + { + while (reader.Read()) + { + switch (reader.Type.LayoutCode) + { + case LayoutCode.TupleScope: + { + ResultAssert.IsSuccess(reader.ReadScope(0, RowReaderUnitTests.ReadTuplePartial)); + break; + } + + case LayoutCode.ObjectScope: + { + ResultAssert.IsSuccess(reader.ReadScope(0, RowReaderUnitTests.ReadNestedDocumentDelegate)); + break; + } + } + } + + return Result.Success; + } + + private static Result ReadNestedDocumentNonDelegate(ref RowReader reader, int context) + { + while (reader.Read()) + { + switch (reader.Type.LayoutCode) + { + case LayoutCode.TupleScope: + { + RowReader nested = reader.ReadScope(); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadTuplePartial(ref nested, 0)); + break; + } + + case LayoutCode.ObjectScope: + { + RowReader nested = reader.ReadScope(); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadNestedDocumentNonDelegate(ref nested, 0)); + ResultAssert.IsSuccess(reader.ReadScope(0, RowReaderUnitTests.ReadNestedDocumentDelegate)); + break; + } + } + } + + return Result.Success; + } + + private static Result ReadNestedDocumentNonDelegateWithSkipScope(ref RowReader reader, int context) + { + while (reader.Read()) + { + switch (reader.Type.LayoutCode) + { + case LayoutCode.TupleScope: + { + RowReader nested = reader.ReadScope(); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadTuplePartial(ref nested, 0)); + ResultAssert.IsSuccess(reader.SkipScope(ref nested)); + break; + } + + case LayoutCode.ObjectScope: + { + RowReader nested = reader.ReadScope(); + ResultAssert.IsSuccess(RowReaderUnitTests.ReadNestedDocumentNonDelegate(ref nested, 0)); + ResultAssert.IsSuccess(reader.ReadScope(0, RowReaderUnitTests.ReadNestedDocumentDelegate)); + ResultAssert.IsSuccess(reader.SkipScope(ref nested)); + break; + } + } + } + + return Result.Success; + } + + private static Result ReadTuplePartial(ref RowReader reader, int unused) + { + // Read only part of our tuple + Assert.IsTrue(reader.Read()); + Assert.IsTrue(reader.Read()); + return Result.Success; + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + internal sealed class Point : IDispatchable, IRowSerializable + { + public readonly int X; + public readonly int Y; + + public Point(int x, int y) + { + this.X = x; + this.Y = y; + } + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Point && this.Equals((Point)obj); + } + + public override int GetHashCode() + { + unchecked + { + return (this.X.GetHashCode() * 397) ^ this.Y.GetHashCode(); + } + } + + Result IRowSerializable.Write(ref RowWriter writer, TypeArgument typeArg) + { + Result result = writer.WriteInt32("x", this.X); + if (result != Result.Success) + { + return result; + } + + return writer.WriteInt32("y", this.Y); + } + + void IDispatchable.Dispatch(ref RowOperationDispatcher dispatcher, ref RowCursor scope) + { + dispatcher.LayoutCodeSwitch(ref scope, "x", value: this.X); + dispatcher.LayoutCodeSwitch(ref scope, "y", value: this.Y); + } + + private bool Equals(Point other) + { + return this.X == other.X && this.Y == other.Y; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/RowWriterUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/RowWriterUnitTests.cs new file mode 100644 index 0000000..b341a4e --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/RowWriterUnitTests.cs @@ -0,0 +1,553 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Buffers; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1118:ParameterMustNotSpanMultipleLines", Justification = "Test code.")] + [DeploymentItem(RowWriterUnitTests.SchemaFile, "TestData")] + public sealed class RowWriterUnitTests + { + private const int InitialRowSize = 2 * 1024 * 1024; + private const string SchemaFile = @"TestData\ReaderSchema.json"; + private static readonly DateTime SampleDateTime = DateTime.Parse("2018-08-14 02:05:00.0000000"); + private static readonly Guid SampleGuid = Guid.Parse("{2A9C25B9-922E-4611-BB0A-244A9496503C}"); + private static readonly Float128 SampleFloat128 = new Float128(0, 42); + private static readonly UnixDateTime SampleUnixDateTime = new UnixDateTime(42); + private static readonly MongoDbObjectId SampleMongoDbObjectId = new MongoDbObjectId(0, 42); + + private Namespace schema; + private LayoutResolver resolver; + + [TestInitialize] + public void TestInitialize() + { + string json = File.ReadAllText(RowWriterUnitTests.SchemaFile); + this.schema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.schema); + } + + [TestMethod] + [Owner("jthunter")] + public void WriteMixed() + { + Layout layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "Mixed").SchemaId); + Assert.IsNotNull(layout); + + RowBuffer row = new RowBuffer(RowWriterUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, layout, this.resolver); + + int writerLength = 0; + ResultAssert.IsSuccess( + RowWriter.WriteBuffer( + ref row, + null, + (ref RowWriter writer, TypeArgument rootTypeArg, object ignored) => + { + ResultAssert.IsSuccess(writer.WriteNull("null")); + ResultAssert.IsSuccess(writer.WriteBool("bool", true)); + ResultAssert.IsSuccess(writer.WriteInt8("int8", (sbyte)-86)); + ResultAssert.IsSuccess(writer.WriteInt16("int16", (short)-21846)); + ResultAssert.IsSuccess(writer.WriteInt32("int32", -1431655766)); + ResultAssert.IsSuccess(writer.WriteInt64("int64", -6148914691236517206L)); + ResultAssert.IsSuccess(writer.WriteUInt8("uint8", (byte)0xAA)); + ResultAssert.IsSuccess(writer.WriteUInt16("uint16", (ushort)0xAAAA)); + ResultAssert.IsSuccess(writer.WriteUInt32("uint32", 0xAAAAAAAA)); + ResultAssert.IsSuccess(writer.WriteUInt64("uint64", 0xAAAAAAAAAAAAAAAAL)); + ResultAssert.IsSuccess(writer.WriteFloat32("float32", 1.0F / 3.0F)); + ResultAssert.IsSuccess(writer.WriteFloat64("float64", 1.0 / 3.0)); + ResultAssert.IsSuccess(writer.WriteFloat128("float128", RowWriterUnitTests.SampleFloat128)); + ResultAssert.IsSuccess(writer.WriteDecimal("decimal", 1.0M / 3.0M)); + ResultAssert.IsSuccess(writer.WriteDateTime("datetime", RowWriterUnitTests.SampleDateTime)); + ResultAssert.IsSuccess(writer.WriteUnixDateTime("unixdatetime", RowWriterUnitTests.SampleUnixDateTime)); + ResultAssert.IsSuccess(writer.WriteGuid("guid", RowWriterUnitTests.SampleGuid)); + ResultAssert.IsSuccess(writer.WriteMongoDbObjectId("mongodbobjectid", RowWriterUnitTests.SampleMongoDbObjectId)); + ResultAssert.IsSuccess(writer.WriteString("utf8", "abc")); + ResultAssert.IsSuccess(writer.WriteString("utf8_span", Utf8Span.TranscodeUtf16("abc"))); + ResultAssert.IsSuccess(writer.WriteBinary("binary", new[] { (byte)0, (byte)1, (byte)2 })); + ResultAssert.IsSuccess(writer.WriteBinary("binary_span", new[] { (byte)0, (byte)1, (byte)2 }.AsSpan())); + ResultAssert.IsSuccess( + writer.WriteBinary("binary_sequence", new ReadOnlySequence(new[] { (byte)0, (byte)1, (byte)2 }))); + + ResultAssert.IsSuccess(writer.WriteVarInt("var_varint", -6148914691236517206L)); + ResultAssert.IsSuccess(writer.WriteVarUInt("var_varuint", 0xAAAAAAAAAAAAAAAAL)); + ResultAssert.IsSuccess(writer.WriteString("var_utf8", "abc")); + ResultAssert.IsSuccess(writer.WriteString("var_utf8_span", Utf8Span.TranscodeUtf16("abc"))); + ResultAssert.IsSuccess(writer.WriteBinary("var_binary", new[] { (byte)0, (byte)1, (byte)2 })); + ResultAssert.IsSuccess(writer.WriteBinary("var_binary_span", new[] { (byte)0, (byte)1, (byte)2 }.AsSpan())); + ResultAssert.IsSuccess( + writer.WriteBinary("var_binary_sequence", new ReadOnlySequence(new[] { (byte)0, (byte)1, (byte)2 }))); + + ResultAssert.IsSuccess(writer.WriteNull("sparse_null")); + ResultAssert.IsSuccess(writer.WriteBool("sparse_bool", true)); + ResultAssert.IsSuccess(writer.WriteInt8("sparse_int8", (sbyte)-86)); + ResultAssert.IsSuccess(writer.WriteInt16("sparse_int16", (short)-21846)); + ResultAssert.IsSuccess(writer.WriteInt32("sparse_int32", -1431655766)); + ResultAssert.IsSuccess(writer.WriteInt64("sparse_int64", -6148914691236517206L)); + ResultAssert.IsSuccess(writer.WriteUInt8("sparse_uint8", (byte)0xAA)); + ResultAssert.IsSuccess(writer.WriteUInt16("sparse_uint16", (ushort)0xAAAA)); + ResultAssert.IsSuccess(writer.WriteUInt32("sparse_uint32", 0xAAAAAAAA)); + ResultAssert.IsSuccess(writer.WriteUInt64("sparse_uint64", 0xAAAAAAAAAAAAAAAAL)); + ResultAssert.IsSuccess(writer.WriteFloat32("sparse_float32", 1.0F / 3.0F)); + ResultAssert.IsSuccess(writer.WriteFloat64("sparse_float64", 1.0 / 3.0)); + ResultAssert.IsSuccess(writer.WriteFloat128("sparse_float128", RowWriterUnitTests.SampleFloat128)); + ResultAssert.IsSuccess(writer.WriteDecimal("sparse_decimal", 1.0M / 3.0M)); + ResultAssert.IsSuccess(writer.WriteDateTime("sparse_datetime", RowWriterUnitTests.SampleDateTime)); + ResultAssert.IsSuccess(writer.WriteUnixDateTime("sparse_unixdatetime", RowWriterUnitTests.SampleUnixDateTime)); + ResultAssert.IsSuccess(writer.WriteGuid("sparse_guid", RowWriterUnitTests.SampleGuid)); + ResultAssert.IsSuccess(writer.WriteMongoDbObjectId("sparse_mongodbobjectid", RowWriterUnitTests.SampleMongoDbObjectId)); + ResultAssert.IsSuccess(writer.WriteString("sparse_utf8", "abc")); + ResultAssert.IsSuccess(writer.WriteString("sparse_utf8_span", Utf8Span.TranscodeUtf16("abc"))); + ResultAssert.IsSuccess(writer.WriteBinary("sparse_binary", new[] { (byte)0, (byte)1, (byte)2 })); + ResultAssert.IsSuccess(writer.WriteBinary("sparse_binary_span", new[] { (byte)0, (byte)1, (byte)2 }.AsSpan())); + ResultAssert.IsSuccess( + writer.WriteBinary("sparse_binary_sequence", new ReadOnlySequence(new[] { (byte)0, (byte)1, (byte)2 }))); + + LayoutColumn col; + Assert.IsTrue(layout.TryFind("array_t", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new sbyte[] { -86, -87, -88 }, + (ref RowWriter writer2, TypeArgument typeArg, sbyte[] values) => + { + foreach (sbyte value in values) + { + ResultAssert.IsSuccess(writer2.WriteInt8(null, value)); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("array_t>", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { new float[] { 1, 2, 3 }, new float[] { 1, 2, 3 } }, + (ref RowWriter writer2, TypeArgument typeArg, float[][] values) => + { + foreach (float[] value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + typeArg.TypeArgs[0], + value, + (ref RowWriter writer3, TypeArgument typeArg2, float[] values2) => + { + foreach (float value2 in values2) + { + ResultAssert.IsSuccess(writer3.WriteFloat32(null, value2)); + } + + return Result.Success; + })); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("array_t", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { "abc", "def", "hij" }, + (ref RowWriter writer2, TypeArgument typeArg, string[] values) => + { + foreach (string value in values) + { + ResultAssert.IsSuccess(writer2.WriteString(null, value)); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("tuple", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + Tuple.Create(-6148914691236517206L, -6148914691236517206L), + (ref RowWriter writer2, TypeArgument typeArg, Tuple values) => + { + ResultAssert.IsSuccess(writer2.WriteVarInt(null, values.Item1)); + ResultAssert.IsSuccess(writer2.WriteInt64(null, values.Item2)); + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("tuple>", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + Tuple.Create(NullValue.Default, Tuple.Create((sbyte)-86, (sbyte)-86)), + (ref RowWriter writer2, TypeArgument typeArg, Tuple> values) => + { + ResultAssert.IsSuccess(writer2.WriteNull(null)); + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + typeArg.TypeArgs[1], + values.Item2, + (ref RowWriter writer3, TypeArgument typeArg2, Tuple values2) => + { + ResultAssert.IsSuccess(writer3.WriteInt8(null, values2.Item1)); + ResultAssert.IsSuccess(writer3.WriteInt8(null, values2.Item2)); + return Result.Success; + })); + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("tuple", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + Tuple.Create(false, new RowReaderUnitTests.Point(1, 2)), + (ref RowWriter writer2, TypeArgument typeArg, Tuple values) => + { + ResultAssert.IsSuccess(writer2.WriteBool(null, values.Item1)); + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + typeArg.TypeArgs[1], + values.Item2, + (ref RowWriter writer3, TypeArgument typeArg2, IRowSerializable values2) => + values2.Write(ref writer3, typeArg2))); + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("nullable", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + Tuple.Create(default(int?), (long?)123L), + (ref RowWriter writer2, TypeArgument typeArg, Tuple values) => + { + RowWriter.WriterFunc f0 = null; + if (values.Item1.HasValue) + { + f0 = (ref RowWriter writer3, TypeArgument typeArg2, int? value) => writer3.WriteInt32(null, value.Value); + } + + ResultAssert.IsSuccess(writer2.WriteScope(null, typeArg.TypeArgs[0], values.Item1, f0)); + + RowWriter.WriterFunc f1 = null; + if (values.Item2.HasValue) + { + f1 = (ref RowWriter writer3, TypeArgument typeArg2, long? value) => writer3.WriteInt64(null, value.Value); + } + + ResultAssert.IsSuccess(writer2.WriteScope(null, typeArg.TypeArgs[1], values.Item2, f1)); + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("tagged", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + Tuple.Create((byte)3, "hello"), + (ref RowWriter writer2, TypeArgument typeArg, Tuple values) => + { + ResultAssert.IsSuccess(writer2.WriteUInt8(null, values.Item1)); + ResultAssert.IsSuccess(writer2.WriteString(null, values.Item2)); + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("tagged", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + Tuple.Create((byte)5, true, "bye"), + (ref RowWriter writer2, TypeArgument typeArg, Tuple values) => + { + ResultAssert.IsSuccess(writer2.WriteUInt8(null, values.Item1)); + ResultAssert.IsSuccess(writer2.WriteBool(null, values.Item2)); + ResultAssert.IsSuccess(writer2.WriteString(null, values.Item3)); + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("set_t", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { "abc", "xzy", "efg" }, + (ref RowWriter writer2, TypeArgument typeArg, string[] values) => + { + foreach (string value in values) + { + ResultAssert.IsSuccess(writer2.WriteString(null, value)); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("set_t>", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { new sbyte[] { 7, 8, 9 }, new sbyte[] { 4, 5, 6 }, new sbyte[] { 1, 2, 3 } }, + (ref RowWriter writer2, TypeArgument typeArg, sbyte[][] values) => + { + foreach (sbyte[] value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + typeArg.TypeArgs[0], + value, + (ref RowWriter writer3, TypeArgument typeArg2, sbyte[] values2) => + { + foreach (sbyte value2 in values2) + { + ResultAssert.IsSuccess(writer3.WriteInt8(null, value2)); + } + + return Result.Success; + })); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("set_t>", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { new[] { 4, 5, 6 }, new[] { 7, 8, 9 }, new[] { 1, 2, 3 } }, + (ref RowWriter writer2, TypeArgument typeArg, int[][] values) => + { + foreach (int[] value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + typeArg.TypeArgs[0], + value, + (ref RowWriter writer3, TypeArgument typeArg2, int[] values2) => + { + foreach (int value2 in values2) + { + ResultAssert.IsSuccess(writer3.WriteInt32(null, value2)); + } + + return Result.Success; + })); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("set_t", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { new RowReaderUnitTests.Point(1, 2), new RowReaderUnitTests.Point(3, 4), new RowReaderUnitTests.Point(5, 6) }, + (ref RowWriter writer2, TypeArgument typeArg, RowReaderUnitTests.Point[] values) => + { + foreach (RowReaderUnitTests.Point value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + typeArg.TypeArgs[0], + value, + (ref RowWriter writer3, TypeArgument typeArg2, IRowSerializable values2) => + values2.Write(ref writer3, typeArg2))); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("map_t", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { Tuple.Create("Harrison", "Han"), Tuple.Create("Mark", "Luke") }, + (ref RowWriter writer2, TypeArgument typeArg, Tuple[] values) => + { + foreach (Tuple value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + new TypeArgument(LayoutType.TypedTuple, typeArg.TypeArgs), + value, + (ref RowWriter writer3, TypeArgument typeArg2, Tuple values2) => + { + ResultAssert.IsSuccess(writer3.WriteString(null, values2.Item1)); + ResultAssert.IsSuccess(writer3.WriteString(null, values2.Item2)); + return Result.Success; + })); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("map_t>", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] { Tuple.Create((sbyte)2, new sbyte[] { 4, 5, 6 }), Tuple.Create((sbyte)1, new sbyte[] { 1, 2, 3 }) }, + (ref RowWriter writer2, TypeArgument typeArg, Tuple[] values) => + { + foreach (Tuple value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + new TypeArgument(LayoutType.TypedTuple, typeArg.TypeArgs), + value, + (ref RowWriter writer3, TypeArgument typeArg2, Tuple values2) => + { + ResultAssert.IsSuccess(writer3.WriteInt8(null, values2.Item1)); + ResultAssert.IsSuccess( + writer3.WriteScope( + null, + typeArg2.TypeArgs[1], + values2.Item2, + (ref RowWriter writer4, TypeArgument typeArg3, sbyte[] values3) => + { + foreach (sbyte value3 in values3) + { + ResultAssert.IsSuccess(writer4.WriteInt8(null, value3)); + } + + return Result.Success; + })); + + return Result.Success; + })); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("map_t>", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] + { + Tuple.Create((short)2, new[] { Tuple.Create(7, 8), Tuple.Create(5, 6) }), + Tuple.Create((short)1, new[] { Tuple.Create(3, 4), Tuple.Create(1, 2) }), + }, + (ref RowWriter writer2, TypeArgument typeArg, Tuple[]>[] values) => + { + foreach (Tuple[]> value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + new TypeArgument(LayoutType.TypedTuple, typeArg.TypeArgs), + value, + (ref RowWriter writer3, TypeArgument typeArg2, Tuple[]> values2) => + { + ResultAssert.IsSuccess(writer3.WriteInt16(null, values2.Item1)); + ResultAssert.IsSuccess( + writer3.WriteScope( + null, + typeArg2.TypeArgs[1], + values2.Item2, + (ref RowWriter writer4, TypeArgument typeArg3, Tuple[] values3) => + { + foreach (Tuple value3 in values3) + { + ResultAssert.IsSuccess( + writer4.WriteScope( + null, + new TypeArgument(LayoutType.TypedTuple, typeArg3.TypeArgs), + value3, + (ref RowWriter writer5, TypeArgument typeArg4, Tuple values4) => + { + ResultAssert.IsSuccess(writer5.WriteInt32(null, values4.Item1)); + ResultAssert.IsSuccess(writer5.WriteInt32(null, values4.Item2)); + return Result.Success; + })); + } + + return Result.Success; + })); + + return Result.Success; + })); + } + + return Result.Success; + })); + + Assert.IsTrue(layout.TryFind("map_t", out col)); + ResultAssert.IsSuccess( + writer.WriteScope( + col.Path, + col.TypeArg, + new[] + { + Tuple.Create(1.0, new RowReaderUnitTests.Point(1, 2)), + Tuple.Create(2.0, new RowReaderUnitTests.Point(3, 4)), + Tuple.Create(3.0, new RowReaderUnitTests.Point(5, 6)), + }, + (ref RowWriter writer2, TypeArgument typeArg, Tuple[] values) => + { + foreach (Tuple value in values) + { + ResultAssert.IsSuccess( + writer2.WriteScope( + null, + new TypeArgument(LayoutType.TypedTuple, typeArg.TypeArgs), + value, + (ref RowWriter writer3, TypeArgument typeArg2, Tuple values2) => + { + ResultAssert.IsSuccess(writer3.WriteFloat64(null, values2.Item1)); + ResultAssert.IsSuccess( + writer3.WriteScope( + null, + typeArg2.TypeArgs[1], + values2.Item2, + (ref RowWriter writer4, TypeArgument typeArg3, IRowSerializable values3) => + values3.Write(ref writer4, typeArg3))); + + return Result.Success; + })); + } + + return Result.Success; + })); + + // Save the RowWriter length after everything is written for later comparison. + writerLength = writer.Length; + return Result.Success; + })); + + RowReader reader = new RowReader(ref row); + Assert.AreEqual(reader.Length, writerLength); + RowReaderUnitTests.PrintReader(ref reader, 0); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/SchemaHashUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/SchemaHashUnitTests.cs new file mode 100644 index 0000000..d91dd64 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/SchemaHashUnitTests.cs @@ -0,0 +1,249 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + [TestClass] + [DeploymentItem(SchemaHashUnitTests.SchemaFile, "TestData")] + public class SchemaHashUnitTests + { + private const string SchemaFile = @"TestData\SchemaHashCoverageSchema.json"; + private Namespace ns; + private Schema tableSchema; + + [TestInitialize] + public void InitializeSuite() + { + string json = File.ReadAllText(SchemaHashUnitTests.SchemaFile); + this.ns = Namespace.Parse(json); + this.tableSchema = this.ns.Schemas.Find(s => s.Name == "Table"); + } + + [TestMethod] + [Owner("jthunter")] + public void SchemaHashCompileTest() + { + Layout layout = this.tableSchema.Compile(this.ns); + Assert.IsNotNull(layout); + } + + [TestMethod] + [Owner("jthunter")] + public void SchemaHashTest() + { + (ulong low, ulong high) hash = SchemaHash.ComputeHash(this.ns, this.tableSchema); + Assert.AreNotEqual((0, 0), hash); + (ulong low, ulong high) hash2 = SchemaHash.ComputeHash(this.ns, this.tableSchema, (1, 1)); + Assert.AreNotEqual(hash, hash2); + + // Test clone are the same. + Schema clone = SchemaHashUnitTests.Clone(this.tableSchema); + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreEqual(hash, hash2); + + // Test Schema changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Name = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreEqual(hash, hash2); // Name not part of the hash + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Comment = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreEqual(hash, hash2); // Comment not part of the hash + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Version = (SchemaLanguageVersion)1; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreEqual(hash, hash2); // Encoding version not part of the hash + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.SchemaId = new SchemaId(42); + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Type = TypeKind.Int8; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Options changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Options.EnablePropertyLevelTimestamp = !clone.Options.EnablePropertyLevelTimestamp; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Options.DisallowUnschematized = !clone.Options.DisallowUnschematized; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Partition Keys changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.PartitionKeys[0].Path = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Primary Sort Keys changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.PrimarySortKeys[0].Path = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.PrimarySortKeys[0].Direction = SortDirection.Descending; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Static Keys changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.StaticKeys[0].Path = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Properties changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Properties[0].Comment = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreEqual(hash, hash2); // Comment not part of the hash + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Properties[0].Path = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Properties[0].PropertyType.ApiType = "something else"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Properties[0].PropertyType.Type = TypeKind.Binary; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + clone.Properties[0].PropertyType.Nullable = !clone.Properties[0].PropertyType.Nullable; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Primitive Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[0].PropertyType as PrimitivePropertyType).Length = 42; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[0].PropertyType as PrimitivePropertyType).Storage = StorageKind.Variable; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Scope Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[1].PropertyType as ScopePropertyType).Immutable = !(clone.Properties[1].PropertyType as ScopePropertyType).Immutable; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Array Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[1].PropertyType as ArrayPropertyType).Items = clone.Properties[0].PropertyType; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Object Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[2].PropertyType as ObjectPropertyType).Properties[0] = clone.Properties[0]; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Map Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[3].PropertyType as MapPropertyType).Keys = clone.Properties[0].PropertyType; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[3].PropertyType as MapPropertyType).Values = clone.Properties[0].PropertyType; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Set Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[4].PropertyType as SetPropertyType).Items = clone.Properties[0].PropertyType; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Tagged Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[5].PropertyType as TaggedPropertyType).Items[0] = clone.Properties[0].PropertyType; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test Tuple Property Type changes + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[6].PropertyType as TuplePropertyType).Items[0] = clone.Properties[0].PropertyType; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Test UDT Property Type changes + try + { + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[7].PropertyType as UdtPropertyType).Name = "some non-existing UDT name"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.Fail("Should have thrown an exception."); + } + catch (Exception ex) + { + Assert.IsNotNull(ex); + } + + try + { + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[7].PropertyType as UdtPropertyType).Name = "Table"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.Fail("Should have thrown an exception."); + } + catch (Exception ex) + { + Assert.IsNotNull(ex); + } + + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[7].PropertyType as UdtPropertyType).Name = "Table"; + (clone.Properties[7].PropertyType as UdtPropertyType).SchemaId = new SchemaId(2); + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreNotEqual(hash, hash2); + + // Renaming an UDT is not a breaking change as long as the SchemaId has not changed. + this.ns.Schemas[0].Name = "RenameActualUDT"; + clone = SchemaHashUnitTests.Clone(this.tableSchema); + (clone.Properties[7].PropertyType as UdtPropertyType).Name = "RenameActualUDT"; + hash2 = SchemaHash.ComputeHash(this.ns, clone); + Assert.AreEqual(hash, hash2); + } + + private static Schema Clone(Schema s) + { + JsonSerializerSettings settings = new JsonSerializerSettings() + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented, + CheckAdditionalContent = true, + }; + + string json = JsonConvert.SerializeObject(s, settings); + return JsonConvert.DeserializeObject(json, settings); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/SchemaIdUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/SchemaIdUnitTests.cs new file mode 100644 index 0000000..8b95fdd --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/SchemaIdUnitTests.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + [TestClass] + public class SchemaIdUnitTests + { + [TestMethod] + [Owner("jthunter")] + public void SchemaIdTest() + { + SchemaId a = new SchemaId(1); + SchemaId b = new SchemaId(2); + SchemaId c = default(SchemaId); + + Assert.AreEqual(1, a.Id); + Assert.AreEqual(2, b.Id); + Assert.AreEqual(SchemaId.Invalid, c); + Assert.AreNotEqual(2, a.Id); + Assert.AreNotEqual(a, b); + Assert.IsTrue(a == a); + Assert.IsTrue(a != b); + Assert.IsFalse(a.Equals(null)); + Assert.AreEqual(a.GetHashCode(), new SchemaId(1).GetHashCode()); + Assert.AreNotEqual(a.GetHashCode(), new SchemaId(-1).GetHashCode()); + + string json = JsonConvert.SerializeObject(a); + Assert.AreEqual("1", json); + Assert.AreEqual("1", a.ToString()); + + Assert.AreEqual(a, JsonConvert.DeserializeObject(json)); + json = JsonConvert.SerializeObject(b); + Assert.AreEqual("2", json); + Assert.AreEqual("2", b.ToString()); + Assert.AreEqual(b, JsonConvert.DeserializeObject(json)); + json = JsonConvert.SerializeObject(c); + Assert.AreEqual("0", json); + Assert.AreEqual("0", c.ToString()); + Assert.AreEqual(c, JsonConvert.DeserializeObject(json)); + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/SchemaUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/SchemaUnitTests.cs new file mode 100644 index 0000000..f548d6e --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/SchemaUnitTests.cs @@ -0,0 +1,603 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + [TestClass] + [SuppressMessage("Naming", "DontUseVarForVariableTypes", Justification = "The types here are anonymous.")] + public class SchemaUnitTests + { + [TestMethod] + [Owner("jthunter")] + public void ParseNamespace() + { + // Test empty schemas. + { + string[] emptyNamespaceJsonInput = + { + @"{ }", + @"{'schemas': [ ] }", + @"{'name': null }", + @"{'name': null, 'schemas': null }", + @"{'version': 'v1', 'name': null, 'schemas': null }", + }; + + foreach (string json in emptyNamespaceJsonInput) + { + Namespace n1 = Namespace.Parse(json); + Assert.IsNull(n1.Name, "Got: {0}, Json: {1}", n1.Name, json); + Assert.IsNotNull(n1.Schemas, "Json: {0}", json); + Assert.AreEqual(0, n1.Schemas.Count, "Got: {0}, Json: {1}", n1.Schemas, json); + Assert.AreEqual(SchemaLanguageVersion.V1, n1.Version); + } + } + + // Test simple schemas and schema options. + { + string json = @"{'name': 'myschema', 'schemas': null }"; + Namespace n1 = Namespace.Parse(json); + Assert.AreEqual("myschema", n1.Name, "Json: {0}", json); + Assert.IsNotNull(n1.Schemas, "Json: {0}", json); + + // Version defaults propertly when NOT specified. + Assert.AreEqual(SchemaLanguageVersion.V1, n1.Version); + } + + { + string json = @"{'name': 'myschema', 'schemas': [ + {'version': 'v1', 'name': 'emptyTable', 'id': -1, 'type': 'schema', + 'options': { 'disallowUnschematized': true }, 'properties': null } ] }"; + + Namespace n1 = Namespace.Parse(json); + Assert.AreEqual("myschema", n1.Name, "Json: {0}", json); + Assert.AreEqual(1, n1.Schemas.Count, "Json: {0}", json); + Assert.AreEqual("emptyTable", n1.Schemas[0].Name, "Json: {0}", json); + Assert.AreEqual(new SchemaId(-1), n1.Schemas[0].SchemaId, "Json: {0}", json); + Assert.AreEqual(TypeKind.Schema, n1.Schemas[0].Type, "Json: {0}", json); + Assert.AreEqual(true, n1.Schemas[0].Options.DisallowUnschematized, "Json: {0}", json); + Assert.IsNotNull(n1.Schemas[0].Properties.Count, "Json: {0}", json); + Assert.AreEqual(0, n1.Schemas[0].Properties.Count, "Json: {0}", json); + Assert.AreEqual(SchemaLanguageVersion.V1, n1.Version); + } + + // Test basic schema with primitive columns. + { + string json = @"{'name': 'myschema', 'schemas': [ + {'name': 'myUDT', 'id': 1, 'type': 'schema', 'options': { 'disallowUnschematized': false }, + 'properties': [ + { 'path': 'a', 'type': { 'type': 'int8', 'storage': 'fixed' }}, + { 'path': 'b', 'type': { 'type': 'utf8', 'storage': 'variable' }} + ] } + ] }"; + + Namespace n1 = Namespace.Parse(json); + Assert.AreEqual(1, n1.Schemas.Count, "Json: {0}", json); + Assert.AreEqual("myUDT", n1.Schemas[0].Name, "Json: {0}", json); + Assert.AreEqual(new SchemaId(1), n1.Schemas[0].SchemaId, "Json: {0}", json); + Assert.AreEqual(TypeKind.Schema, n1.Schemas[0].Type, "Json: {0}", json); + Assert.AreEqual(false, n1.Schemas[0].Options.DisallowUnschematized, "Json: {0}", json); + Assert.AreEqual(2, n1.Schemas[0].Properties.Count, "Json: {0}", json); + + var expectedProps = new[] + { + new { Path = "a", Type = TypeKind.Int8, Storage = StorageKind.Fixed }, + new { Path = "b", Type = TypeKind.Utf8, Storage = StorageKind.Variable }, + }; + + for (int i = 0; i < n1.Schemas[0].Properties.Count; i++) + { + Property p = n1.Schemas[0].Properties[i]; + Assert.AreEqual(expectedProps[i].Path, p.Path, "Json: {0}", json); + Assert.AreEqual(expectedProps[i].Type, p.PropertyType.Type, "Json: {0}", json); + PrimitivePropertyType sp = (PrimitivePropertyType)p.PropertyType; + Assert.AreEqual(expectedProps[i].Storage, sp.Storage, "Json: {0}", json); + } + } + } + + [TestMethod] + [Owner("jthunter")] + [DeploymentItem(@"TestData\CoverageSchema.json", "TestData")] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(@"TestData\CoverageSchema.json"); + Namespace n1 = Namespace.Parse(json); + JsonSerializerSettings settings = new JsonSerializerSettings() + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented, + }; + + string json2 = JsonConvert.SerializeObject(n1, settings); + Namespace n2 = Namespace.Parse(json2); + string json3 = JsonConvert.SerializeObject(n2, settings); + Assert.AreEqual(json2, json3); + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaPrimitives() + { + // Test all primitive column types. + dynamic[] expectedSchemas = + { + new { Json = @"{'type': 'bool', 'storage': 'fixed'}", Type = TypeKind.Boolean }, + + new { Json = @"{'type': 'int8', 'storage': 'fixed'}", Type = TypeKind.Int8 }, + new { Json = @"{'type': 'int16', 'storage': 'fixed'}", Type = TypeKind.Int16 }, + new { Json = @"{'type': 'int32', 'storage': 'fixed'}", Type = TypeKind.Int32 }, + new { Json = @"{'type': 'int64', 'storage': 'fixed'}", Type = TypeKind.Int64 }, + new { Json = @"{'type': 'uint8', 'storage': 'fixed'}", Type = TypeKind.UInt8 }, + new { Json = @"{'type': 'uint16', 'storage': 'fixed'}", Type = TypeKind.UInt16 }, + new { Json = @"{'type': 'uint32', 'storage': 'fixed'}", Type = TypeKind.UInt32 }, + new { Json = @"{'type': 'uint64', 'storage': 'fixed'}", Type = TypeKind.UInt64 }, + + new { Json = @"{'type': 'float32', 'storage': 'fixed'}", Type = TypeKind.Float32 }, + new { Json = @"{'type': 'float64', 'storage': 'fixed'}", Type = TypeKind.Float64 }, + new { Json = @"{'type': 'float128', 'storage': 'fixed'}", Type = TypeKind.Float128 }, + new { Json = @"{'type': 'decimal', 'storage': 'fixed'}", Type = TypeKind.Decimal }, + + new { Json = @"{'type': 'datetime', 'storage': 'fixed'}", Type = TypeKind.DateTime }, + new { Json = @"{'type': 'unixdatetime', 'storage': 'fixed'}", Type = TypeKind.UnixDateTime }, + + new { Json = @"{'type': 'guid', 'storage': 'fixed'}", Type = TypeKind.Guid }, + new { Json = @"{'type': 'mongodbobjectid', 'storage': 'fixed'}", Type = TypeKind.MongoDbObjectId }, + + new { Json = @"{'type': 'varint', 'storage': 'variable'}", Type = TypeKind.VarInt }, + new { Json = @"{'type': 'varuint', 'storage': 'variable'}", Type = TypeKind.VarUInt }, + + new { Json = @"{'type': 'utf8', 'storage': 'fixed', 'length': 2}", Type = TypeKind.Utf8, Len = 2 }, + new { Json = @"{'type': 'binary', 'storage': 'fixed', 'length': 2}", Type = TypeKind.Binary, Len = 2 }, + new { Json = @"{'type': 'utf8', 'storage': 'variable', 'length': 100}", Type = TypeKind.Utf8, Len = 100 }, + new { Json = @"{'type': 'binary', 'storage': 'variable', 'length': 100}", Type = TypeKind.Binary, Len = 100 }, + new { Json = @"{'type': 'utf8', 'sparse': 'variable', 'length': 1000}", Type = TypeKind.Utf8, Len = 1000 }, + new { Json = @"{'type': 'binary', 'sparse': 'variable', 'length': 1000}", Type = TypeKind.Binary, Len = 1000 }, + }; + + foreach (dynamic expected in expectedSchemas) + { + string columnSchema = $"{{'path': 'a', 'type': {expected.Json}}}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{columnSchema}]}}"; + Schema s = Schema.Parse(tableSchema); + Assert.AreEqual(1, s.Properties.Count, "Json: {0}", expected.Json); + Property p = s.Properties[0]; + Assert.AreEqual(expected.Type, p.PropertyType.Type, "Json: {0}", expected.Json); + PrimitivePropertyType sp = (PrimitivePropertyType)p.PropertyType; + switch (p.PropertyType.Type) + { + case TypeKind.Utf8: + case TypeKind.Binary: + switch (sp.Storage) + { + case StorageKind.Fixed: + case StorageKind.Variable: + case StorageKind.Sparse: + Assert.AreEqual(expected.Len, sp.Length, "Json: {0}", expected.Json); + break; + default: + Assert.Fail("Json: {0}", expected.Json); + break; + } + + break; + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaArray() + { + // Test array types include nested arrays. + dynamic[] expectedSchemas = + { + new { Json = @"{'type': 'int8' }", Type = TypeKind.Int8 }, + new { Json = @"{'type': 'array', 'items': {'type': 'int32'}}", Type = TypeKind.Int32 }, + new { Json = @"{'type': 'object', 'properties': null}", Len = 0 }, + new { Json = @"{'type': 'schema', 'name': 'myUDT'}", Name = "myUDT" }, + }; + + foreach (dynamic expected in expectedSchemas) + { + string arrayColumnSchema = $"{{'path': 'a', 'type': {{'type': 'array', 'items': {expected.Json} }} }}"; + string tableSchema = $"{{'name': 'table', 'id': -1, 'type': 'schema', 'properties': [{arrayColumnSchema}] }}"; + Schema s = Schema.Parse(tableSchema); + Assert.AreEqual(1, s.Properties.Count, "Json: {0}", expected.Json); + Property p = s.Properties[0]; + Assert.AreEqual(TypeKind.Array, p.PropertyType.Type, "Json: {0}", expected.Json); + ArrayPropertyType pt = (ArrayPropertyType)p.PropertyType; + Assert.IsNotNull(pt.Items, "Json: {0}", expected.Json); + switch (pt.Items.Type) + { + case TypeKind.Array: + ArrayPropertyType subArray = (ArrayPropertyType)pt.Items; + Assert.AreEqual(expected.Type, subArray.Items.Type, "Json: {0}", expected.Json); + break; + case TypeKind.Object: + ObjectPropertyType subObj = (ObjectPropertyType)pt.Items; + Assert.AreEqual(expected.Len, subObj.Properties.Count, "Json: {0}", expected.Json); + break; + case TypeKind.Schema: + UdtPropertyType subRow = (UdtPropertyType)pt.Items; + Assert.AreEqual(expected.Name, subRow.Name, "Json: {0}", expected.Json); + break; + default: + Assert.AreEqual(expected.Type, pt.Items.Type, "Json: {0}", expected.Json); + break; + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaObject() + { + // Test object types include nested objects. + dynamic[] expectedSchemas = + { + new { Json = @"{'path': 'b', 'type': {'type': 'int8', 'storage': 'fixed'}}", Type = TypeKind.Int8 }, + new { Json = @"{'path': 'b', 'type': {'type': 'array', 'items': {'type': 'int32'}}}", Type = TypeKind.Int32 }, + new { Json = @"{'path': 'b', 'type': {'type': 'object', 'properties': [{'path': 'c', 'type': {'type': 'bool'}}]}}", Len = 1 }, + new { Json = @"{'path': 'b', 'type': {'type': 'schema', 'name': 'myUDT'}}", Name = "myUDT" }, + }; + + foreach (dynamic expected in expectedSchemas) + { + string objectColumnSchema = $"{{'path': 'a', 'type': {{'type': 'object', 'properties': [{expected.Json}] }} }}"; + string tableSchema = $"{{'name': 'table', 'id': -2, 'type': 'schema', 'properties': [{objectColumnSchema}] }}"; + try + { + Schema s = Schema.Parse(tableSchema); + Assert.AreEqual(1, s.Properties.Count, "Json: {0}", expected.Json); + Property pa = s.Properties[0]; + Assert.AreEqual(TypeKind.Object, pa.PropertyType.Type, "Json: {0}", expected.Json); + ObjectPropertyType pt = (ObjectPropertyType)pa.PropertyType; + Assert.AreEqual(1, pt.Properties.Count, "Json: {0}", expected.Json); + Property pb = pt.Properties[0]; + switch (pb.PropertyType.Type) + { + case TypeKind.Array: + ArrayPropertyType subArray = (ArrayPropertyType)pb.PropertyType; + Assert.AreEqual(expected.Type, subArray.Items.Type, "Json: {0}", expected.Json); + break; + case TypeKind.Object: + ObjectPropertyType subObj = (ObjectPropertyType)pb.PropertyType; + Assert.AreEqual(expected.Len, subObj.Properties.Count, "Json: {0}", expected.Json); + break; + case TypeKind.Schema: + UdtPropertyType subRow = (UdtPropertyType)pb.PropertyType; + Assert.AreEqual(expected.Name, subRow.Name, "Json: {0}", expected.Json); + break; + default: + Assert.AreEqual(expected.Type, pb.PropertyType.Type, "Json: {0}", expected.Json); + break; + } + } + catch (Exception ex) + { + Assert.Fail("Exception: {0}, Json: {1}", ex, expected.Json); + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaPartitionPrimaryKeys() + { + // Test parsing both partition and primary sort keys. + var expectedSchemas = new[] + { + new + { + JsonPK = @"{'path': 'a'}", + JsonCK = @"{'path': 'b', 'direction': 'desc'}, {'path': 'c'}", + PK = new[] { "a" }, + CK = new[] { new { Path = "b", Dir = SortDirection.Descending }, new { Path = "c", Dir = SortDirection.Ascending } }, + }, + }; + + foreach (var expected in expectedSchemas) + { + string tableSchema = $@"{{ + 'name': 'table', + 'id': -3, + 'type': 'schema', + 'properties': [ + {{'path': 'a', 'type': {{'type': 'int8', 'storage': 'fixed'}}}}, + {{'path': 'b', 'type': {{'type': 'utf8', 'storage': 'variable', 'length': 2}}}}, + {{'path': 'c', 'type': {{'type': 'datetime', 'storage': 'fixed'}}}}, + ], + 'partitionkeys': [{expected.JsonPK}], + 'primarykeys': [{expected.JsonCK}] + }}"; + + try + { + Schema s = Schema.Parse(tableSchema); + Assert.AreEqual(3, s.Properties.Count, "PK: {0}, CK: {1}", expected.JsonPK, expected.JsonCK); + for (int i = 0; i < s.PartitionKeys.Count; i++) + { + Assert.AreEqual(expected.PK[i], s.PartitionKeys[i].Path, "PK: {0}, CK: {1}", expected.JsonPK, expected.JsonCK); + } + + for (int i = 0; i < s.PrimarySortKeys.Count; i++) + { + Assert.AreEqual(expected.CK[i].Path, s.PrimarySortKeys[i].Path, "PK: {0}, CK: {1}", expected.JsonPK, expected.JsonCK); + Assert.AreEqual(expected.CK[i].Dir, s.PrimarySortKeys[i].Direction, "PK: {0}, CK: {1}", expected.JsonPK, expected.JsonCK); + } + } + catch (Exception ex) + { + Assert.Fail("Exception: {0}, PK: {1}, CK: {2}", ex, expected.JsonPK, expected.JsonCK); + } + } + } + + [TestMethod] + [Owner("vahemesw")] + public void ParseSchemaStaticKeys() + { + var expectedSchemas = new[] + { + new + { + NumberOfPaths = 1, + JsonStaticKeys = @"{'path': 'c'}", + StaticKeys = new[] { new { Path = "c" } }, + }, + new + { + NumberOfPaths = 2, + JsonStaticKeys = @"{'path': 'c'}, {'path': 'd'}", + StaticKeys = new[] { new { Path = "c" }, new { Path = "d" } }, + }, + }; + + foreach (var expected in expectedSchemas) + { + string tableSchema = $@"{{ + 'name': 'table', + 'id': -3, + 'type': 'schema', + 'properties': [ + {{'path': 'a', 'type': {{'type': 'int8', 'storage': 'fixed'}}}}, + {{'path': 'b', 'type': {{'type': 'utf8', 'storage': 'variable', 'length': 2}}}}, + {{'path': 'c', 'type': {{'type': 'datetime', 'storage': 'fixed'}}}}, + {{'path': 'd', 'type': {{'type': 'int8', 'storage': 'fixed'}}}}, + ], + 'partitionkeys': [{{'path': 'a'}}], + 'primarykeys': [{{'path': 'b', 'direction': 'desc'}}], + 'statickeys': [{expected.JsonStaticKeys}] + }}"; + + try + { + Schema s = Schema.Parse(tableSchema); + Assert.AreEqual(expected.NumberOfPaths, s.StaticKeys.Count); + for (int i = 0; i < s.StaticKeys.Count; i++) + { + Assert.AreEqual(expected.StaticKeys[i].Path, s.StaticKeys[i].Path); + } + } + catch (Exception ex) + { + Assert.Fail("Exception: {0}, Caught exception when deserializing the schema", ex); + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void ParseSchemaApiType() + { + // Test api type specifications include elements of complex types. + var expectedSchemas = new[] + { + new { Json = @"{'type': 'int64', 'apitype': 'counter'}", ApiType = "counter" }, + new { Json = @"{'type': 'array', 'items': {'type': 'int64', 'apitype': 'timestamp'}}", ApiType = "timestamp" }, + }; + + foreach (var expected in expectedSchemas) + { + string columnSchema = $"{{'path': 'a', 'type': {expected.Json}}}"; + string tableSchema = $"{{'name': 'table', 'id': -4, 'type': 'schema', 'properties': [{columnSchema}]}}"; + Schema s = Schema.Parse(tableSchema); + Assert.AreEqual(1, s.Properties.Count, "Json: {0}", expected.Json); + Property p = s.Properties[0]; + switch (p.PropertyType.Type) + { + case TypeKind.Array: + ArrayPropertyType subArray = (ArrayPropertyType)p.PropertyType; + Assert.AreEqual(expected.ApiType, subArray.Items.ApiType, "Json: {0}", expected.Json); + break; + default: + Assert.AreEqual(expected.ApiType, p.PropertyType.ApiType, "Json: {0}", expected.Json); + break; + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void SchemaRef() + { + NamespaceParserTest[] tests = new NamespaceParserTest[] + { + new NamespaceParserTest + { + Name = "SchemaNameOnlyRef", + Json = @"{'schemas': [ + { 'name': 'A', 'id': 1, 'type': 'schema'}, + { 'name': 'B', 'id': 2, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'A'}} + ]} + ]}", + }, + new NamespaceParserTest + { + Name = "SchemaNameAndIdRef", + Json = @"{'schemas': [ + { 'name': 'A', 'id': 1, 'type': 'schema'}, + { 'name': 'B', 'id': 2, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'A', 'id': 1}} + ]} + ]}", + }, + new NamespaceParserTest + { + Name = "SchemaMultipleVersionNameAndIdRef", + Json = @"{'schemas': [ + { 'name': 'A', 'id': 1, 'type': 'schema'}, + { 'name': 'B', 'id': 2, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'A', 'id': 3}} + ]}, + { 'name': 'A', 'id': 3, 'type': 'schema'} + ]}", + }, + }; + + foreach (NamespaceParserTest t in tests) + { + Console.WriteLine(t.Name); + Namespace.Parse(t.Json); + } + } + + [TestMethod] + [Owner("jthunter")] + public void NegativeNamespaceParser() + { + NamespaceParserTest[] tests = new NamespaceParserTest[] + { + new NamespaceParserTest + { + Name = "InvalidId", + Json = @"{'schemas': [{ 'name': 'A', 'id': 0, 'type': 'schema'}]}", + }, + new NamespaceParserTest + { + Name = "InvalidNameEmpty", + Json = @"{'schemas': [{ 'name': '', 'id': 1, 'type': 'schema'}]}", + }, + new NamespaceParserTest + { + Name = "InvalidNameWhitespace", + Json = @"{'schemas': [{ 'name': ' ', 'id': 1, 'type': 'schema'}]}", + }, + new NamespaceParserTest + { + Name = "DuplicateId", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema'}, { 'name': 'B', 'id': 1, 'type': 'schema'} ]}", + }, + new NamespaceParserTest + { + Name = "DuplicatePropertyName", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'bool'}}, + {'path': 'b', 'type': {'type': 'int8'}}, + ]}]}", + }, + new NamespaceParserTest + { + Name = "MissingPK", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'partitionkeys': [{'path': 'b'}]}]}", + }, + new NamespaceParserTest + { + Name = "MissingPS", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'primarykeys': [{'path': 'b'}]}]}", + }, + new NamespaceParserTest + { + Name = "MissingStaticKey", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'statickeys': [{'path': 'b'}]}]}", + }, + new NamespaceParserTest + { + Name = "InvalidPropertyName", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [{'path': '', 'type': {'type': 'bool'}}]}]}", + }, + new NamespaceParserTest + { + Name = "InvalidLength", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'utf8', 'length': -1}} + ]}]}", + }, + new NamespaceParserTest + { + Name = "InvalidStorage", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'array', 'items': {'type': 'utf8', 'storage': 'fixed'}}} + ]}]}", + }, + new NamespaceParserTest + { + Name = "DuplicateObjectProperties", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'object', 'properties': [ + {'path': 'c', 'type': {'type': 'bool'}}, + {'path': 'c', 'type': {'type': 'int8'}} + ]}} + ]}]}", + }, + new NamespaceParserTest + { + Name = "MissingUDTName", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'B'}} + ]}]}", + }, + new NamespaceParserTest + { + Name = "MissingUDTId", + Json = @"{'schemas': [{ 'name': 'A', 'id': 1, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'A', 'id': 3}} + ]}]}", + }, + new NamespaceParserTest + { + Name = "MismatchedSchemaRef", + Json = @"{'schemas': [ + { 'name': 'A', 'id': 1, 'type': 'schema'}, + { 'name': 'B', 'id': 2, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'A', 'id': 2}} + ]} + ]}", + }, + new NamespaceParserTest + { + Name = "AmbiguousSchemaRef", + Json = @"{'schemas': [ + { 'name': 'A', 'id': 1, 'type': 'schema'}, + { 'name': 'B', 'id': 2, 'type': 'schema', 'properties': [ + {'path': 'b', 'type': {'type': 'schema', 'name': 'A'}} + ]}, + { 'name': 'A', 'id': 3, 'type': 'schema'} + ]}", + }, + }; + + foreach (NamespaceParserTest t in tests) + { + Console.WriteLine(t.Name); + AssertThrowsException.ThrowsException(() => Namespace.Parse(t.Json)); + } + } + + private struct NamespaceParserTest + { + public string Name { get; set; } + + public string Json { get; set; } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/SerializerUnitTest.cs b/dotnet/src/HybridRow.Tests.Unit/SerializerUnitTest.cs new file mode 100644 index 0000000..38d6eb6 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/SerializerUnitTest.cs @@ -0,0 +1,317 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1401 // Fields should be private +#pragma warning disable SA1201 // Elements should appear in the correct order +#pragma warning disable SA1204 // Elements should appear in the correct order +#pragma warning disable CA1034 // Nested types should not be visible +#pragma warning disable CA1051 // Do not declare visible instance fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(SerializerUnitTest.SchemaFile, "TestData")] + public sealed class SerializerUnitTest + { + private const string SchemaFile = @"TestData\BatchApiSchema.json"; + private const int InitialRowSize = 2 * 1024 * 1024; + + private Namespace schema; + private LayoutResolver resolver; + private Layout layout; + + [TestInitialize] + public void InitTestSuite() + { + string json = File.ReadAllText(SerializerUnitTest.SchemaFile); + this.schema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.schema); + this.layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "BatchRequest").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateBatchRequest() + { + BatchRequest request = new BatchRequest() + { + Operations = new List() + { + new BatchOperation() + { + OperationType = 3, + Headers = new BatchRequestHeaders() + { + SampleRequestHeader = 12345L, + }, + ResourceType = 1, + ResourcePath = "/some/url/path", + ResourceBody = new byte[] { 1, 2, 3 }, + }, + new BatchOperation() + { + OperationType = 2, + Headers = new BatchRequestHeaders() + { + SampleRequestHeader = 98746L, + }, + ResourceType = 2, + ResourcePath = "/some/other/url/path", + ResourceBody = new byte[] { 3, 2, 1 }, + }, + }, + }; + + // Write the request by serializing it to a row. + RowBuffer row = new RowBuffer(SerializerUnitTest.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + Result r = RowWriter.WriteBuffer(ref row, request, BatchRequestSerializer.Write); + Assert.AreEqual(Result.Success, r); + Console.WriteLine("Length of serialized row: {0}", row.Length); + + // Read the row back again. + RowReader reader = new RowReader(ref row); + r = BatchRequestSerializer.Read(ref reader, out BatchRequest _); + Assert.AreEqual(Result.Success, r); + + // Dump the materialized request to the console. + reader = new RowReader(ref row); + r = DiagnosticConverter.ReaderToString(ref reader, out string dumpStr); + Assert.AreEqual(Result.Success, r); + Console.WriteLine(dumpStr); + } + + public sealed class BatchRequestHeaders + { + public long SampleRequestHeader; + } + + public sealed class BatchOperation + { + public int OperationType; + public BatchRequestHeaders Headers; + public int ResourceType; + public string ResourcePath; + public byte[] ResourceBody; + } + + public sealed class BatchRequest + { + public List Operations; + } + + public sealed class BatchResponseHeaders + { + public string SampleResponseHeader; + } + + public sealed class BatchOperationResponse + { + public int StatusCode; + public BatchResponseHeaders Headers; + public byte[] ResourceBody; + } + + public sealed class BatchResponse + { + public List Operations; + } + + public static class BatchRequestHeadersSerializer + { + public static readonly TypeArgument TypeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(new SchemaId(1))); + + public static Result Write(ref RowWriter writer, TypeArgument typeArg, BatchRequestHeaders header) + { + Result r = writer.WriteInt64("sampleRequestHeader", header.SampleRequestHeader); + if (r != Result.Success) + { + return r; + } + + return Result.Success; + } + + public static Result Read(ref RowReader reader, out BatchRequestHeaders header) + { + BatchRequestHeaders retval = new BatchRequestHeaders(); + header = default; + while (reader.Read()) + { + switch (reader.Path) + { + case "sampleRequestHeader": + Result r = reader.ReadInt64(out retval.SampleRequestHeader); + if (r != Result.Success) + { + return r; + } + + break; + } + } + + header = retval; + return Result.Success; + } + } + + public static class BatchOperationSerializer + { + public static readonly TypeArgument TypeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(new SchemaId(2))); + + public static Result Write(ref RowWriter writer, TypeArgument typeArg, BatchOperation operation) + { + Result r = writer.WriteInt32("operationType", operation.OperationType); + if (r != Result.Success) + { + return r; + } + + r = writer.WriteScope("headers", BatchRequestHeadersSerializer.TypeArg, operation.Headers, BatchRequestHeadersSerializer.Write); + if (r != Result.Success) + { + return r; + } + + r = writer.WriteInt32("resourceType", operation.ResourceType); + if (r != Result.Success) + { + return r; + } + + r = writer.WriteString("resourcePath", operation.ResourcePath); + if (r != Result.Success) + { + return r; + } + + r = writer.WriteBinary("resourceBody", operation.ResourceBody); + if (r != Result.Success) + { + return r; + } + + return Result.Success; + } + + public static Result Read(ref RowReader reader, out BatchOperation operation) + { + BatchOperation retval = new BatchOperation(); + operation = default; + while (reader.Read()) + { + Result r; + switch (reader.Path) + { + case "operationType": + r = reader.ReadInt32(out retval.OperationType); + if (r != Result.Success) + { + return r; + } + + break; + case "headers": + r = reader.ReadScope( + retval, + (ref RowReader child, BatchOperation parent) => + BatchRequestHeadersSerializer.Read(ref child, out parent.Headers)); + if (r != Result.Success) + { + return r; + } + + break; + case "resourceType": + r = reader.ReadInt32(out retval.ResourceType); + if (r != Result.Success) + { + return r; + } + + break; + case "resourcePath": + r = reader.ReadString(out retval.ResourcePath); + if (r != Result.Success) + { + return r; + } + + break; + case "resourceBody": + r = reader.ReadBinary(out retval.ResourceBody); + if (r != Result.Success) + { + return r; + } + + break; + } + } + + operation = retval; + return Result.Success; + } + } + + public static class BatchRequestSerializer + { + public static readonly TypeArgument OperationsTypeArg = new TypeArgument( + LayoutType.TypedArray, + new TypeArgumentList(new[] { BatchOperationSerializer.TypeArg })); + + public static Result Write(ref RowWriter writer, TypeArgument typeArg, BatchRequest request) + { + return writer.WriteScope( + "operations", + BatchRequestSerializer.OperationsTypeArg, + request.Operations, + (ref RowWriter writer2, TypeArgument typeArg2, List operations) => + { + foreach (BatchOperation operation in operations) + { + Result r = writer2.WriteScope(null, BatchOperationSerializer.TypeArg, operation, BatchOperationSerializer.Write); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + }); + } + + public static Result Read(ref RowReader reader, out BatchRequest request) + { + Assert.IsTrue(reader.Read()); + Contract.Assert(reader.Path == "operations"); + + Result r = reader.ReadList(BatchOperationSerializer.Read, out List operations); + if (r != Result.Success) + { + request = default; + return r; + } + + request = new BatchRequest() + { + Operations = operations, + }; + + return Result.Success; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TaggedUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/TaggedUnitTests.cs new file mode 100644 index 0000000..2e3e0e0 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TaggedUnitTests.cs @@ -0,0 +1,145 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(TaggedUnitTests.SchemaFile, "TestData")] + public sealed class TaggedUnitTests + { + private const string SchemaFile = @"TestData\TaggedApiSchema.json"; + private const int InitialRowSize = 2 * 1024 * 1024; + + private Namespace schema; + private LayoutResolver resolver; + private Layout layout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(TaggedUnitTests.SchemaFile); + this.schema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.schema); + this.layout = this.resolver.Resolve(this.schema.Schemas.Find(x => x.Name == "TaggedApi").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateTaggedApi() + { + RowBuffer row = new RowBuffer(TaggedUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + TaggedApi c1 = new TaggedApi() + { + Tag1 = ((byte)1, "hello"), + Tag2 = ((byte)2, 28, 1974L), + }; + + this.WriteTaggedApi(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + TaggedApi c2 = this.ReadTaggedApi(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(c1, c2); + } + + private void WriteTaggedApi(ref RowBuffer row, ref RowCursor root, TaggedApi pc) + { + Assert.IsTrue(this.layout.TryFind("tag1", out LayoutColumn c)); + root.Clone(out RowCursor tag1Scope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref tag1Scope, c.TypeArgs, out tag1Scope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tag1Scope, pc.Tag1.Item1)); + Assert.IsTrue(tag1Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref tag1Scope, pc.Tag1.Item2)); + + Assert.IsTrue(this.layout.TryFind("tag2", out c)); + root.Clone(out RowCursor tag2Scope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref tag2Scope, c.TypeArgs, out tag2Scope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tag2Scope, pc.Tag2.Item1)); + Assert.IsTrue(tag2Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref tag2Scope, pc.Tag2.Item2)); + Assert.IsTrue(tag2Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[2].Type.TypeAs().WriteSparse(ref row, ref tag2Scope, pc.Tag2.Item3)); + } + + private TaggedApi ReadTaggedApi(ref RowBuffer row, ref RowCursor root) + { + TaggedApi pc = new TaggedApi(); + + Assert.IsTrue(this.layout.TryFind("tag1", out LayoutColumn c)); + Assert.IsTrue(c.Type.Immutable); + root.Clone(out RowCursor tag1Scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref tag1Scope, out tag1Scope) == Result.Success) + { + Assert.IsTrue(tag1Scope.Immutable); + Assert.IsTrue(tag1Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tag1Scope, out byte apiCode)); + Assert.IsTrue(tag1Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().ReadSparse(ref row, ref tag1Scope, out string str)); + pc.Tag1 = (apiCode, str); + } + + Assert.IsTrue(this.layout.TryFind("tag2", out c)); + Assert.IsFalse(c.Type.Immutable); + root.Clone(out RowCursor tag2Scope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref tag2Scope, out tag2Scope) == Result.Success) + { + Assert.IsFalse(tag2Scope.Immutable); + Assert.IsTrue(tag2Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tag2Scope, out byte apiCode)); + Assert.IsTrue(tag2Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().ReadSparse(ref row, ref tag2Scope, out int val1)); + Assert.IsTrue(tag2Scope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[2].Type.TypeAs().ReadSparse(ref row, ref tag2Scope, out long val2)); + pc.Tag2 = (apiCode, val1, val2); + } + + return pc; + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class TaggedApi + { + public (byte, string) Tag1; + public (byte, int, long) Tag2; + + // ReSharper disable once MemberCanBePrivate.Local + public bool Equals(TaggedApi other) + { + return object.Equals(this.Tag1, other.Tag1) && + object.Equals(this.Tag2, other.Tag2); + } + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is TaggedApi taggedApi && this.Equals(taggedApi); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = 0; + hashCode = (hashCode * 397) ^ this.Tag1.GetHashCode(); + hashCode = (hashCode * 397) ^ this.Tag2.GetHashCode(); + return hashCode; + } + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/BatchApiSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/BatchApiSchema.json new file mode 100644 index 0000000..7922591 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/BatchApiSchema.json @@ -0,0 +1,96 @@ +{ + "name": "Microsoft.Azure.Cosmos.BatchApi", + "version": "v1", + "schemas": [ + { + "name": "BatchRequestHeaders", + "id": 1, + "type": "schema", + "properties": + [ + { + "path": "sampleRequestHeader", + "type": { "type": "int64", "storage": "fixed" } + } + ] + }, + { + "name": "BatchOperation", + "id": 2, + "type": "schema", + "properties": + [ + { + "path": "operationType", + "type": { "type": "int32", "storage": "fixed" } + }, { + "path": "headers", + "type": { "type": "schema", "name": "BatchRequestHeaders" } + }, { + "path": "resourceType", + "type": { "type": "int32", "storage": "fixed" } + }, { + "path": "resourcePath", + "type": { "type": "utf8", "storage": "variable", "length": 1024 } + }, { + "path": "resourceBody", + "type": { "type": "binary", "storage": "variable" } + } + ] + }, + { + "name": "BatchRequest", + "id": 3, + "type": "schema", + "properties": + [ + { + "path": "operations", + "type": { "type": "array", "items": { "type": "schema", "name": "BatchOperation" } } + } + ] + }, + { + "name": "BatchResponseHeaders", + "id": 4, + "type": "schema", + "properties": + [ + { + "path": "sampleResponseHeader", + "type": { "type": "utf8", "storage": "variable", "length": 1024 } + } + ] + }, + { + "name": "BatchOperationResponse", + "id": 5, + "type": "schema", + "properties": + [ + { + "path": "statusCode", + "type": { "type": "int32", "storage": "fixed" } + }, { + "path": "headers", + "type": { "type": "schema", "name": "BatchResponseHeaders" } + }, { + "path": "resourceBody", + "type": { "type": "binary", "storage": "variable" } + } + ] + }, + { + "name": "BatchResponse", + "id": 6, + "type": "schema", + "properties": + [ + { + "path": "operations", + "type": { "type": "array", "items": { "type": "schema", "name": "BatchOperationResponse" } } + } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/CoverageSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/CoverageSchema.json new file mode 100644 index 0000000..70ab311 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/CoverageSchema.json @@ -0,0 +1,211 @@ +{ + "schemas": [ + { + "name": "myUDT", // Question: what should the namespace/structure of schema identifiers. + "id": 1, + "type": "schema", // Optional: implied at the top-level (only "schema" types can be defined at the root of schemas) + "options": { + "disallowUnschematized": false // Optional: defaults to false + }, + "properties": [ + { + "path": "a", + "type": { + "type": "int8", + "storage": "fixed" + } + }, + { + "path": "b", + "type": { + "type": "utf8", + "storage": "variable" + } + } + ] + }, + { + "name": "someTable", + "id": -1, + "options": { + "disallowUnschematized": true + }, + "properties": [ + { + "path": "myBool", + "comment": "A sample fixed boolean column", + "type": { + "type": "bool", + "storage": "fixed" + } + }, + { + "path": "myInt8", + "comment": "A sample fixed 8-byte integer column", + "type": { + "type": "int8", + "storage": "fixed" + } + }, + { + "path": "nested.x", + "comment": "A sample nested integer column", + "type": { + "type": "int32", + "storage": "fixed" + } + }, + { + "path": "nested.y", + "comment": "A sample nested float column", + "type": { + "type": "float32", + "storage": "fixed" + } + }, + { + "path": "nested.deeper.z", + "comment": "A sample deeper nested double column", + "type": { + "type": "float64", + "storage": "fixed" + } + }, + { + "path": "State", + "comment": "A sample fixed 2-byte UTF-8 encoded text column", + "type": { + "type": "utf8", + "storage": "fixed", + "length": 2 + } + }, + { + "path": "myString", + "comment": "A sample variable length UTF-8 encoded text column (up to 127 bytes)", + "type": { + "type": "utf8", + "storage": "variable" + } + }, + { + "path": "lob", + "comment": "A sample extended UTF-8 encoded text column (up to 2M bytes)", + "type": { + "type": "utf8", + "storage": "sparse" + } + }, + { + "path": "canbelob", + "comment": + "A sample extended UTF-8 encoded text column (up to 2M bytes) that stores variable if 'small' (<127 bytes), but sparse if 'large'", + "type": { + "type": "utf8", + "storage": "variable" + } + }, + { + "path": "primitiveArray", + "comment": "A sample array of primitives (4-byte ints)", + "type": { "type": "array", "items": { "type": "int32" } } + }, + { + "path": "shreddedArray[0]", + "comment": "A sample fixed-length array of primitives", + "type": { "type": "int32" } + }, + { + "path": "shreddedArray[1]", + "comment": "A sample fixed-length array of primitives", + "type": { "type": "int32" } + }, + { + "path": "nestedArray", + "comment": "A sample array of nested arrays", + "type": { + "type": "array", + "items": { + "type": "array", + "items": { "type": "int32" } + } + } + }, + { + "path": "nestedNestedArray", + "comment": "A sample array of nested nested arrays", + "type": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { "type": "int32" } + } + } + } + }, + { + "path": "arrayOfObject", + "comment": "A sample array of semi-structured objects", + "type": { + "type": "array", + "items": { + "type": "object", + "properties": [ + { + "path": "a", + "type": { "type": "int8" } + }, + { + "path": "b", + "type": { "type": "utf8" } + } + ] + } + } + }, + { + "path": "arrayOfAny", + "comment": "A sample heterogenous array", + "type": { + "type": "array", + "items": { "type": "any" } + } + }, + { + "path": "arrayOfUDT", + "comment": "A sample array of schematized rows", + "type": { + "type": "array", + "items": { + "type": "schema", + "name": "myUDT" // see definition above - should this be called $ref or ref or something? + } + } + }, + { + "path": "nestedObject", + "comment": "A sample nested objects", + "type": { + "type": "object", + "properties": [ + { + "path": "a", + "type": { + "type": "int8" + } + }, + { + "path": "b", + "type": { + "type": "utf8" + } + } + ] + } + } + ] + } + ] +} \ No newline at end of file diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/CrossVersioningExpected.json b/dotnet/src/HybridRow.Tests.Unit/TestData/CrossVersioningExpected.json new file mode 100644 index 0000000..4628eff --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/CrossVersioningExpected.json @@ -0,0 +1,8 @@ +{ + "CrossVersionFixed": "8101000000FFFF1FAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAA3E555555555555D53F2A00000000000000000000000000000000001C00CA44C50A55555505CB00B714006E39578A01D6082A00000000000000B9259C2A2E921146BB0A244A9496503C2A0000000000000000000000616263000102", + "CrossVersionNullFixed": "810100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "CrossVersionVariable": "81020000000FADD5AAD5AAD5AAD5AA01AAD5AAD5AAD5AAD5AA010361626303000102", + "CrossVersionNullVariable": "810200000000", + "CrossVersionSparse": "8103000000010103020503AA0604AAAA0705AAAAAAAA0806AAAAAAAAAAAAAAAA0907AA0A08AAAA0B09AAAAAAAA0C0AAAAAAAAAAAAAAAAA0F0BABAAAA3E100C555555555555D53F160D2A000000000000000000000000000000110E00001C00CA44C50A55555505CB00B714120F006E39578A01D60817102A000000000000001311B9259C2A2E921146BB0A244A9496503C18122A000000000000000000000014130361626315140300010222051503000000AAAAAA22220F1602000000030000000000803F0000004000004040030000000000803F00000040000040402214170300000003616263036465660368696A26020D0818ADD5AAD5AAD5AAD5AA01AAAAAAAAAAAAAAAA260201260205051901AAAA26020344040000001A02030100000002000000462E141B03000000036162630365666703787A792E22051C030000000300000001020303000000040506030000000708092E2E071D030000000300000001000000020000000300000003000000040000000500000006000000030000000700000008000000090000002E44040000001E030000000301000000020000004603030000000400000046030500000006000000462A14141F02000000044D61726B044C756B65084861727269736F6E0348616E2A0522052002000000010300000001020302030000000405062A062A0707210200000001000200000001000000020000000300000004000000020002000000050000000600000007000000080000002A1044040000002203000000000000000000004003030000000400000046000000000000084003050000000600000046000000000000F03F03010000000200000046", + "CrossVersionNullSparse": "8103000000" +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/CrossVersioningSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/CrossVersioningSchema.json new file mode 100644 index 0000000..e5f9f0a --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/CrossVersioningSchema.json @@ -0,0 +1,127 @@ +// Set of types used in the cross versioning tests. +{ + "schemas": [ + { "name": "Fixed", "id": 1, "type": "schema", + "properties": [ + { "path": "null", "type": { "type": "null", "storage": "fixed" } }, + { "path": "bool", "type": { "type": "bool", "storage": "fixed" } }, + { "path": "int8", "type": { "type": "int8", "storage": "fixed" } }, + { "path": "int16", "type": { "type": "int16", "storage": "fixed" } }, + { "path": "int32", "type": { "type": "int32", "storage": "fixed" } }, + { "path": "int64", "type": { "type": "int64", "storage": "fixed" } }, + { "path": "uint8", "type": { "type": "uint8", "storage": "fixed" } }, + { "path": "uint16", "type": { "type": "uint16", "storage": "fixed" } }, + { "path": "uint32", "type": { "type": "uint32", "storage": "fixed" } }, + { "path": "uint64", "type": { "type": "uint64", "storage": "fixed" } }, + { "path": "float32", "type": { "type": "float32", "storage": "fixed" } }, + { "path": "float64", "type": { "type": "float64", "storage": "fixed" } }, + { "path": "float128", "type": { "type": "float128", "storage": "fixed" } }, + { "path": "decimal", "type": { "type": "decimal", "storage": "fixed" } }, + { "path": "datetime", "type": { "type": "datetime", "storage": "fixed" } }, + { "path": "unixdatetime", "type": { "type": "unixdatetime", "storage": "fixed" } }, + { "path": "guid", "type": { "type": "guid", "storage": "fixed" } }, + { "path": "mongodbobjectid", "type": { "type": "mongodbobjectid", "storage": "fixed" } }, + { "path": "utf8", "type": { "type": "utf8", "storage": "fixed", "length": 3 } }, + { "path": "binary", "type": { "type": "binary", "storage": "fixed", "length": 3 } } + ]}, + { "name": "Variable", "id": 2, "type": "schema", + "properties": [ + { "path": "varint", "type": { "type": "varint", "storage": "variable" } }, + { "path": "varuint", "type": { "type": "varuint", "storage": "variable" } }, + { "path": "utf8", "type": { "type": "utf8", "storage": "variable"} }, + { "path": "binary", "type": { "type": "binary", "storage": "variable" } } + ]}, + { "name": "Sparse", "id": 3, "type": "schema", + "properties": [ + { "path": "null", "type": { "type": "null" } }, + { "path": "bool", "type": { "type": "bool" } }, + { "path": "int8", "type": { "type": "int8" } }, + { "path": "int16", "type": { "type": "int16" } }, + { "path": "int32", "type": { "type": "int32" } }, + { "path": "int64", "type": { "type": "int64" } }, + { "path": "uint8", "type": { "type": "uint8" } }, + { "path": "uint16", "type": { "type": "uint16" } }, + { "path": "uint32", "type": { "type": "uint32" } }, + { "path": "uint64", "type": { "type": "uint64" } }, + { "path": "float32", "type": { "type": "float32" } }, + { "path": "float64", "type": { "type": "float64" } }, + { "path": "float128", "type": { "type": "float128" } }, + { "path": "decimal", "type": { "type": "decimal" } }, + { "path": "datetime", "type": { "type": "datetime" } }, + { "path": "unixdatetime", "type": { "type": "unixdatetime" } }, + { "path": "guid", "type": { "type": "guid" } }, + { "path": "mongodbobjectid", "type": { "type": "mongodbobjectid" } }, + { "path": "utf8", "type": { "type": "utf8" } }, + { "path": "binary", "type": { "type": "binary" } }, + { "path": "array_t", "type": { + "type": "array", + "items": { "type": "int8", "nullable": false } + } }, + { "path": "array_t>", "type": { + "type": "array", + "items": { "type": "array", "nullable": false, "items": { "type": "float32", "nullable": false } } + } }, + { "path": "array_t", "type": { "type": "array", "items": { "type": "utf8", "nullable": false } } }, + { "path": "tuple", "type": { + "type": "tuple", + "items": [ { "type": "varint", "nullable": false }, { "type": "int64", "nullable": false }] + } }, + { "path": "tuple>", "type": { + "type": "tuple", "items": [ + { "type": "null", "nullable": false }, + { "type": "tuple", "nullable": false, "items": [ { "type": "int8", "nullable": false }, { "type": "int8", "nullable": false } ] } + ]}}, + { "path": "tuple", "type": { + "type": "tuple", "items": [ + { "type": "bool", "nullable": false }, + { "type": "schema", "name": "Point", "nullable": false} + ]}}, + { "path": "set_t", "type": { + "type": "set", + "items": { "type": "utf8", "nullable": false } + } }, + { "path": "set_t>", "type": { + "type": "set", + "items": { "type": "array", "nullable": false, "items": { "type": "int8", "nullable": false } } + } }, + { "path": "set_t>", "type": { + "type": "set", + "items": { "type": "set", "nullable": false, "items": { "type": "int32", "nullable": false } } + } }, + { "path": "set_t", "type": { + "type": "set", + "items": { "type": "schema", "name": "Point", "nullable": false} + } }, + { "path": "map_t", "type": { + "type": "map", + "keys": { "type": "utf8", "nullable": false }, + "values": { "type": "utf8", "nullable": false } + } }, + { "path": "map_t>", "type": { + "type": "map", + "keys": { "type": "int8", "nullable": false }, + "values": { "type": "array", "nullable": false, "items": { "type": "int8", "nullable": false } } + } }, + { "path": "map_t>", "type": { + "type": "map", + "keys": { "type": "int16", "nullable": false }, + "values": { + "type": "map", + "nullable": false, + "keys": { "type": "int32", "nullable": false }, + "values": { "type": "int32", "nullable": false } + } + } }, + { "path": "map_t", "type": { + "type": "map", + "keys": { "type": "float64", "nullable": false }, + "values": { "type": "schema", "name": "Point", "nullable": false} + } } + ]}, + { "name": "Point", "id": 4, "type": "schema", + "properties": [ + { "path": "x", "type": { "type": "int32", "storage": "fixed" } }, + { "path": "y", "type": { "type": "int32", "storage": "fixed" } } + ]} + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/CustomerSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/CustomerSchema.json new file mode 100644 index 0000000..1777ca2 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/CustomerSchema.json @@ -0,0 +1,76 @@ +// Partial implementation of Cassandra Hotel Schema described here:: +// https://www.oreilly.com/ideas/cassandra-data-modeling +{ + "name": "Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.CustomerSchema", + "schemas": [ + { + "name": "PostalCode", + "id": 1, + "type": "schema", + "properties": [ + { "path": "zip", "type": { "type": "int32", "storage": "fixed" } }, + { "path": "plus4", "type": { "type": "int16", "storage": "sparse" } } + ] + }, + { + "name": "Address", + "id": 2, + "type": "schema", + "properties": [ + { "path": "street", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "city", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "state", "type": { "type": "utf8", "storage": "fixed", "length": 2 } }, + { "path": "postal_code", "type": { "type": "schema", "name": "PostalCode" } } + ] + }, + { + "name": "Hotels", + "id": 3, + "type": "schema", + "partitionkeys": [{ "path": "hotel_id" }], + "properties": [ + { "path": "hotel_id", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "name", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "phone", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "address", "type": { "type": "schema", "name": "Address", "immutable": true } } + ] + }, + { + "name": "Available_Rooms_By_Hotel_Date", + "id": 4, + "type": "schema", + "partitionkeys": [{ "path": "hotel_id" }], + "primarykeys": [{ "path": "date" }, { "path": "room_number", "direction": "desc" }], + "properties": [ + { "path": "hotel_id", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "date", "type": { "type": "datetime", "storage": "fixed" } }, + { "path": "room_number", "type": { "type": "uint8", "storage": "fixed" } }, + { "path": "is_available", "type": { "type": "bool" } } + ] + }, + { + "name": "Guests", + "id": 5, + "type": "schema", + "partitionkeys": [{ "path": "guest_id" }], + "primarykeys": [{ "path": "first_name" }, { "path": "phone_numbers", "direction": "desc" }], + "properties": [ + { "path": "guest_id", "type": { "type": "guid", "storage": "fixed" } }, + { "path": "first_name", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "last_name", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "title", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "emails", "type": { "type": "array", "items": { "type": "utf8", "nullable": false } } }, + { "path": "phone_numbers", "type": { "type": "array", "items": { "type": "utf8", "nullable": false } } }, + { + "path": "addresses", + "type": { + "type": "map", + "keys": { "type": "utf8", "nullable": false }, + "values": { "type": "schema", "name": "Address", "immutable": true, "nullable": false } + } + }, + { "path": "confirm_number", "type": { "type": "utf8", "storage": "variable" } } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/MovieSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/MovieSchema.json new file mode 100644 index 0000000..11a6730 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/MovieSchema.json @@ -0,0 +1,40 @@ +// Todo demo schema that utilizes typed maps. +{ + "schemas": [ + { "name": "Movie", "id": 1, "type": "schema", + "properties": [ + { "path": "cast", "type": { + "type": "map", + "keys": { "type": "utf8", "nullable": false }, + "values": { "type": "utf8", "nullable": false } + } }, + { "path": "stats", "type": { + "type": "map", + "keys": { "type": "guid", "nullable": false }, + "values": { "type": "float64", "nullable": false } + } }, + { "path": "related", "type": { + "comment": "map: actor -> { map: moveId -> roleName }", + "type": "map", + "keys": { "type": "utf8", "nullable": false }, + "values": { + "type": "map", + "nullable": false, + "keys": { "type": "int64", "nullable": false }, + "values": { "type": "utf8", "nullable": false } + } + } }, + { "path": "revenue", "type": { + "comment": "map: releaseDate -> Earnings }", + "type": "map", + "keys": { "type": "datetime", "nullable": false }, + "values": { "type": "schema", "name": "Earnings", "nullable": false } + } } + ]}, + { "name": "Earnings", "id": 2, "type": "schema", + "properties": [ + { "path": "domestic", "type": { "type": "decimal", "storage": "fixed" } }, + { "path": "worldwide", "type": { "type": "decimal", "storage": "fixed" } } + ]} + ] +} \ No newline at end of file diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/NullableSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/NullableSchema.json new file mode 100644 index 0000000..cf55e42 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/NullableSchema.json @@ -0,0 +1,26 @@ +// Demo schema that utilizes nullable typed scopes. +{ + "schemas": [ + { + "name": "Nullables", + "id": 1, + "type": "schema", + "properties": [ + { "path": "nullbool", "type": { "type": "array", "items": { "type": "bool" } } }, + { "path": "nullset", "type": { "type": "set", "items": { "type": "utf8" } } }, + { "path": "nullarray", "type": { "type": "array", "items": { "type": "float32" } } }, + { + "path": "nulltuple", + "type": { + "type": "array", + "items": { "type": "tuple", "nullable": false, "items": [{ "type": "int32" }, { "type": "int64" }] } + } + }, + { + "path": "nullmap", + "type": { "type": "map", "keys": { "type": "guid" }, "values": { "type": "uint8" } } + } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/PerfCounterSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/PerfCounterSchema.json new file mode 100644 index 0000000..8bb24d7 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/PerfCounterSchema.json @@ -0,0 +1,72 @@ +// Performance Counter demo schema that utilizes tuples. +{ + "schemas": [ + { + "name": "Coord", + "id": 2, + "type": "schema", + "properties": [ + { "path": "lat", "type": { "type": "int64", "storage": "fixed" } }, + { "path": "lng", "type": { "type": "int64", "storage": "fixed" } } + ] + }, + { + "name": "Counters", + "id": 1, + "type": "schema", + "partitionkeys": [{ "path": "name" }], + "properties": [ + { "path": "name", "type": { "type": "utf8", "storage": "variable" } }, + { + "path": "value", + "type": { + "type": "tuple", + "immutable": true, + "items": [{ "type": "utf8", "nullable": false }, { "type": "int64", "nullable": false }] + } + }, + { + "path": "minmeanmax", + "type": { + "type": "tuple", + "immutable": true, + "items": [ + { "type": "utf8", "nullable": false }, + { + "type": "tuple", + "nullable": false, + "items": [ + { "type": "int64", "nullable": false }, + { "type": "int64", "nullable": false }, + { "type": "int64", "nullable": false } + ] + } + ] + } + }, + { + "path": "coord", + "type": { + "type": "tuple", + "immutable": true, + "items": [{ "type": "utf8", "nullable": false }, { "type": "schema", "name": "Coord", "nullable": false }] + } + } + ] + }, + { + "name": "CounterSet", + "id": 3, + "type": "schema", + "properties": [ + { + "path": "history", + "type": { + "type": "array", + "items": { "type": "schema", "name": "Counters", "nullable": false } + } + } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/ReaderSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/ReaderSchema.json new file mode 100644 index 0000000..69f08b6 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/ReaderSchema.json @@ -0,0 +1,143 @@ +// Set of types used in the IO tests. +{ + "schemas": [ + { "name": "Mixed", "id": 1, "type": "schema", + "properties": [ + { "path": "null", "type": { "type": "null", "storage": "fixed" } }, + { "path": "bool", "type": { "type": "bool", "storage": "fixed" } }, + { "path": "int8", "type": { "type": "int8", "storage": "fixed" } }, + { "path": "int16", "type": { "type": "int16", "storage": "fixed" } }, + { "path": "int32", "type": { "type": "int32", "storage": "fixed" } }, + { "path": "int64", "type": { "type": "int64", "storage": "fixed" } }, + { "path": "uint8", "type": { "type": "uint8", "storage": "fixed" } }, + { "path": "uint16", "type": { "type": "uint16", "storage": "fixed" } }, + { "path": "uint32", "type": { "type": "uint32", "storage": "fixed" } }, + { "path": "uint64", "type": { "type": "uint64", "storage": "fixed" } }, + { "path": "float32", "type": { "type": "float32", "storage": "fixed" } }, + { "path": "float64", "type": { "type": "float64", "storage": "fixed" } }, + { "path": "float128", "type": { "type": "float128", "storage": "fixed" } }, + { "path": "decimal", "type": { "type": "decimal", "storage": "fixed" } }, + { "path": "datetime", "type": { "type": "datetime", "storage": "fixed" } }, + { "path": "unixdatetime", "type": { "type": "unixdatetime", "storage": "fixed" } }, + { "path": "guid", "type": { "type": "guid", "storage": "fixed" } }, + { "path": "mongodbobjectid", "type": { "type": "mongodbobjectid", "storage": "fixed" } }, + { "path": "utf8", "type": { "type": "utf8", "storage": "fixed", "length": 3 } }, + { "path": "utf8_span", "type": { "type": "utf8", "storage": "fixed", "length": 3 } }, + { "path": "binary", "type": { "type": "binary", "storage": "fixed", "length": 3 } }, + { "path": "binary_span", "type": { "type": "binary", "storage": "fixed", "length": 3 } }, + { "path": "binary_sequence", "type": { "type": "binary", "storage": "fixed", "length": 3 } }, + { "path": "var_varint", "type": { "type": "varint", "storage": "variable" } }, + { "path": "var_varuint", "type": { "type": "varuint", "storage": "variable" } }, + { "path": "var_utf8", "type": { "type": "utf8", "storage": "variable"} }, + { "path": "var_utf8_span", "type": { "type": "utf8", "storage": "variable"} }, + { "path": "var_binary", "type": { "type": "binary", "storage": "variable" } }, + { "path": "var_binary_span", "type": { "type": "binary", "storage": "variable" } }, + { "path": "var_binary_sequence", "type": { "type": "binary", "storage": "variable" } }, + { "path": "sparse_null", "type": { "type": "null" } }, + { "path": "sparse_bool", "type": { "type": "bool" } }, + { "path": "sparse_int8", "type": { "type": "int8" } }, + { "path": "sparse_int16", "type": { "type": "int16" } }, + { "path": "sparse_int32", "type": { "type": "int32" } }, + { "path": "sparse_int64", "type": { "type": "int64" } }, + { "path": "sparse_uint8", "type": { "type": "uint8" } }, + { "path": "sparse_uint16", "type": { "type": "uint16" } }, + { "path": "sparse_uint32", "type": { "type": "uint32" } }, + { "path": "sparse_uint64", "type": { "type": "uint64" } }, + { "path": "sparse_float32", "type": { "type": "float32" } }, + { "path": "sparse_float64", "type": { "type": "float64" } }, + { "path": "sparse_float128", "type": { "type": "float128" } }, + { "path": "sparse_decimal", "type": { "type": "decimal" } }, + { "path": "sparse_datetime", "type": { "type": "datetime" } }, + { "path": "sparse_unixdatetime", "type": { "type": "unixdatetime" } }, + { "path": "sparse_guid", "type": { "type": "guid" } }, + { "path": "sparse_mongodbobjectid", "type": { "type": "mongodbobjectid" } }, + { "path": "sparse_utf8", "type": { "type": "utf8" } }, + { "path": "sparse_utf8_span", "type": { "type": "utf8" } }, + { "path": "sparse_binary", "type": { "type": "binary" } }, + { "path": "sparse_binary_span", "type": { "type": "binary" } }, + { "path": "sparse_binary_sequence", "type": { "type": "binary" } }, + { "path": "array_t", "type": { + "type": "array", + "items": { "type": "int8", "nullable": false } + } }, + { "path": "array_t>", "type": { + "type": "array", + "items": { "type": "array", "nullable": false, "items": { "type": "float32", "nullable": false } } + } }, + { "path": "array_t", "type": { "type": "array", "items": { "type": "utf8", "nullable": false } } }, + { "path": "tuple", "type": { + "type": "tuple", + "items": [ { "type": "varint", "nullable": false }, { "type": "int64", "nullable": false }] + } }, + { "path": "tuple>", "type": { + "type": "tuple", "items": [ + { "type": "null", "nullable": false }, + { "type": "tuple", "nullable": false, "items": [ { "type": "int8", "nullable": false }, { "type": "int8", "nullable": false } ] } + ]}}, + { "path": "tuple", "type": { + "type": "tuple", "items": [ + { "type": "bool", "nullable": false }, + { "type": "schema", "name": "Point", "nullable": false} + ]}}, + { "path": "nullable", "type": { + "type": "tuple", + "items": [ { "type": "int32", "nullable": true }, { "type": "int64", "nullable": true }] + } }, + { "path": "tagged", "type": { + "type": "tagged", "items": [ + { "type": "utf8", "nullable": false } + ]}}, + { "path": "tagged", "type": { + "type": "tagged", "items": [ + { "type": "bool", "nullable": false }, + { "type": "utf8", "nullable": false } + ]}}, + { "path": "set_t", "type": { + "type": "set", + "items": { "type": "utf8", "nullable": false } + } }, + { "path": "set_t>", "type": { + "type": "set", + "items": { "type": "array", "nullable": false, "items": { "type": "int8", "nullable": false } } + } }, + { "path": "set_t>", "type": { + "type": "set", + "items": { "type": "set", "nullable": false, "items": { "type": "int32", "nullable": false } } + } }, + { "path": "set_t", "type": { + "type": "set", + "items": { "type": "schema", "name": "Point", "nullable": false} + } }, + { "path": "map_t", "type": { + "type": "map", + "keys": { "type": "utf8", "nullable": false }, + "values": { "type": "utf8", "nullable": false } + } }, + { "path": "map_t>", "type": { + "type": "map", + "keys": { "type": "int8", "nullable": false }, + "values": { "type": "array", "nullable": false, "items": { "type": "int8", "nullable": false } } + } }, + { "path": "map_t>", "type": { + "type": "map", + "keys": { "type": "int16", "nullable": false }, + "values": { + "type": "map", + "nullable": false, + "keys": { "type": "int32", "nullable": false }, + "values": { "type": "int32", "nullable": false } + } + } }, + { "path": "map_t", "type": { + "type": "map", + "keys": { "type": "float64", "nullable": false }, + "values": { "type": "schema", "name": "Point", "nullable": false} + } } + ]}, + { "name": "Point", "id": 4, "type": "schema", + "properties": [ + { "path": "x", "type": { "type": "int32", "storage": "fixed" } }, + { "path": "y", "type": { "type": "int32", "storage": "fixed" } } + ]} + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/SchemaHashCoverageSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/SchemaHashCoverageSchema.json new file mode 100644 index 0000000..37b423b --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/SchemaHashCoverageSchema.json @@ -0,0 +1,171 @@ +{ + "name": "Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit.SchemaHashTest", + "schemas": [ + { + "version": "v1", + "comment": "Some UDT definition", + "name": "UDT", + "id": 1, + "type": "schema", + "properties": [ + { + "path": "item1", + "type": { + "length": 0, + "storage": "fixed", + "type": "int32", + "nullable": false + } + }, + { + "path": "item2", + "type": { + "length": 10, + "storage": "variable", + "type": "utf8" + } + } + ], + "partitionkeys": [], + "primarykeys": [] + }, + { + "version": "v1", + "comment": "Some table definition", + "name": "Table", + "id": 2, + "options": { + "disallowUnschematized": true, + "enablePropertyLevelTimestamp": true + }, + "type": "schema", + "properties": [ + { + "path": "fixed", + "type": { + "length": 0, + "storage": "fixed", + "apitype": "myfixed", + "type": "int32" + } + }, + { + "path": "array", + "type": { + "items": { + "length": 0, + "storage": "sparse", + "type": "int8" + }, + "immutable": true, + "type": "array" + } + }, + { + "path": "obj", + "type": { + "properties": [ + { + "path": "nested", + "type": { + "length": 0, + "storage": "sparse", + "type": "int32" + } + } + ], + "immutable": false, + "type": "object" + } + }, + { + "path": "map", + "type": { + "keys": { + "length": 0, + "storage": "sparse", + "type": "int8" + }, + "values": { + "length": 0, + "storage": "sparse", + "type": "int8" + }, + "immutable": false, + "type": "map" + } + }, + { + "path": "set", + "type": { + "items": { + "length": 0, + "storage": "sparse", + "type": "int8" + }, + "immutable": false, + "type": "set" + } + }, + { + "path": "tagged", + "type": { + "items": [ + { + "length": 0, + "storage": "sparse", + "type": "int32" + } + ], + "immutable": false, + "type": "tagged" + } + }, + { + "path": "tuple", + "type": { + "items": [ + { + "length": 0, + "storage": "sparse", + "type": "int32" + }, + { + "length": 0, + "storage": "sparse", + "type": "float32" + } + ], + "immutable": false, + "type": "tuple" + } + }, + { + "path": "udt", + "type": { + "name": "UDT", + "id": 1, + "immutable": false, + "type": "schema" + } + } + ], + "partitionkeys": [ + { + "path": "fixed" + } + ], + "primarykeys": [ + { + "path": "fixed", + "direction": "asc" + } + ], + "statickeys": [ + { + "path": "fixed" + } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/TagSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/TagSchema.json new file mode 100644 index 0000000..fcc1b07 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/TagSchema.json @@ -0,0 +1,28 @@ +// Tag demo schema that utilizes typed arrays. +{ + "schemas": [ + { "name": "Tagged", "id": 1, "type": "schema", + "properties": [ + { "path": "title", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "tags", "type": { "type": "array", + "items": { "type": "utf8", "nullable": false } } }, + { "path": "options", "type": { "type": "array", + "items": { "type": "int32", "nullable": true } } }, + { "path": "ratings", "type": { "type": "array", + "items": { "type": "array", "nullable": false, "items": { "type": "float64", "nullable": false } } } }, + { "path": "similars", "type": { "type": "array", + "items": { "type": "schema", "name": "SimilarMatch", "nullable": false } } }, + { "path": "priority", "type": { "type": "array", + "items": { "type": "tuple", "nullable": false, + "items": [ + { "type": "utf8", "nullable": false }, + { "type": "int64", "nullable": false } + ]}}} + ]}, + { "name": "SimilarMatch", "id": 2, "type": "schema", + "properties": [ + { "path": "thumbprint", "type": { "type": "utf8", "storage": "fixed", "length": 18 } }, + { "path": "score", "type": { "type": "float64", "storage": "fixed" } } + ]} + ] +} \ No newline at end of file diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/TaggedApiSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/TaggedApiSchema.json new file mode 100644 index 0000000..0bafcf1 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/TaggedApiSchema.json @@ -0,0 +1,28 @@ +// Tagged demo schema that utilizes tagged types. +{ + "schemas": [ + { + "name": "TaggedApi", + "id": 1, + "type": "schema", + "properties": [ + { + "path": "tag1", + "type": { + "type": "tagged", + "immutable": true, + "items": [{ "type": "utf8", "nullable": false }] + } + }, + { + "path": "tag2", + "type": { + "type": "tagged", + "immutable": false, + "items": [{ "type": "int32", "nullable": false }, { "type": "int64", "nullable": false }] + } + } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TestData/TodoSchema.json b/dotnet/src/HybridRow.Tests.Unit/TestData/TodoSchema.json new file mode 100644 index 0000000..cb91662 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TestData/TodoSchema.json @@ -0,0 +1,75 @@ +// Todo demo schema that utilizes typed sets. +{ + "schemas": [ + { + "name": "Todo", + "id": 1, + "type": "schema", + "properties": [ + { "path": "attendees", "type": { "type": "set", "items": { "type": "utf8", "nullable": false } } }, + { "path": "projects", "type": { "type": "set", "items": { "type": "guid", "nullable": false } } }, + { "path": "checkboxes", "type": { "type": "set", "items": { "type": "bool", "nullable": false } } }, + { + "path": "prices", + "type": { + "type": "set", + "items": { + "type": "set", + "immutable": true, + "nullable": false, + "items": { "type": "float32", "nullable": false } + } + } + }, + { + "path": "nested", + "type": { + "type": "set", + "items": { + "type": "set", + "immutable": true, + "nullable": false, + "items": + { + "type": "set", + "immutable": true, + "nullable": false, + "items": { + "type": "int32", + "nullable": false + } + } + } + } + }, + { + "path": "shopping", + "type": { "type": "set", "items": { "type": "schema", "name": "ShoppingItem", "nullable": false } } + }, + { + "path": "work", + "type": { + "type": "set", + "items": { + "type": "tuple", + "nullable": false, + "items": [ + { "type": "bool", "nullable": false }, + { "type": "varuint", "nullable": false } + ] + } + } + } + ] + }, + { + "name": "ShoppingItem", + "id": 2, + "type": "schema", + "properties": [ + { "path": "label", "type": { "type": "utf8", "storage": "variable" } }, + { "path": "count", "type": { "type": "uint8", "storage": "fixed" } } + ] + } + ] +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TupleUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/TupleUnitTests.cs new file mode 100644 index 0000000..63adcee --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TupleUnitTests.cs @@ -0,0 +1,546 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + // ReSharper disable once StringLiteralTypo + [TestClass] + [SuppressMessage("Naming", "DontUseVarForVariableTypes", Justification = "The types here are anonymous.")] + [DeploymentItem(@"TestData\PerfCounterSchema.json", "TestData")] + public sealed class TupleUnitTests + { + private const int InitialRowSize = 2 * 1024 * 1024; + + private readonly PerfCounter counterExample = new PerfCounter() + { + Name = "RowInserts", + Value = Tuple.Create("units", 12046L), + }; + + private Namespace counterSchema; + private LayoutResolver countersResolver; + private Layout countersLayout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(@"TestData\PerfCounterSchema.json"); + this.counterSchema = Namespace.Parse(json); + this.countersResolver = new LayoutResolverNamespace(this.counterSchema); + this.countersLayout = this.countersResolver.Resolve(this.counterSchema.Schemas.Find(x => x.Name == "Counters").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = this.counterExample; + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + PerfCounter c2 = this.ReadCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(c1, c2); + } + + [TestMethod] + [Owner("jthunter")] + public void VerifyTypeConstraintsCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = this.counterExample; + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + + Assert.IsTrue(this.countersLayout.TryFind("value", out LayoutColumn c)); + RowCursor.Create(ref row, out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + ResultAssert.TypeConstraint(LayoutType.Boolean.WriteSparse(ref row, ref valueScope, true)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, "millis")); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.TypeConstraint(LayoutType.Float32.WriteSparse(ref row, ref valueScope, 0.1F)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref valueScope, 100L)); + } + + [TestMethod] + [Owner("jthunter")] + public void PreventInsertsAndDeletesInFixedArityCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = this.counterExample; + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + + Assert.IsTrue(this.countersLayout.TryFind("value", out LayoutColumn c)); + RowCursor.Create(ref row, out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + RowCursor.Create(ref row, out RowCursor valueScope2).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref valueScope2, out valueScope2)); + Assert.AreEqual(valueScope.AsReadOnly(out RowCursor _).ScopeType, valueScope2.ScopeType); + Assert.AreEqual(valueScope.AsReadOnly(out RowCursor _).start, valueScope2.start); + Assert.AreEqual(valueScope.AsReadOnly(out RowCursor _).Immutable, valueScope2.Immutable); + + ResultAssert.TypeConstraint( + c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, "millis", UpdateOptions.InsertAt)); + ResultAssert.TypeConstraint(c.TypeArgs[0].Type.TypeAs().DeleteSparse(ref row, ref valueScope)); + Assert.IsFalse(valueScope.MoveTo(ref row, 2)); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateMinMeanMaxCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = new PerfCounter() + { + Name = "RowInserts", + MinMaxValue = Tuple.Create("units", Tuple.Create(12L, 542L, 12046L)), + }; + + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + PerfCounter c2 = this.ReadCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(c1, c2); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateCoordCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = new PerfCounter() + { + Name = "CoordInserts", + Coord = Tuple.Create("units", new Coord { Lat = 12L, Lng = 40L }), + }; + + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + PerfCounter c2 = this.ReadCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(c1, c2); + } + + [TestMethod] + [Owner("jthunter")] + public void VerifyTypeConstraintsMinMeanMaxCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = new PerfCounter() + { + Name = "RowInserts", + MinMaxValue = Tuple.Create("units", Tuple.Create(12L, 542L, 12046L)), + }; + + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + + // ReSharper disable once StringLiteralTypo + Assert.IsTrue(this.countersLayout.TryFind("minmeanmax", out LayoutColumn c)); + RowCursor.Create(ref row, out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + ResultAssert.TypeConstraint(LayoutType.DateTime.WriteSparse(ref row, ref valueScope, DateTime.Now)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, "secs")); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.TypeConstraint(LayoutType.Decimal.WriteSparse(ref row, ref valueScope, 12M)); + + TypeArgument mmmType = c.TypeArgs[1]; + + // Invalid because not a tuple type. + ResultAssert.TypeConstraint( + mmmType.Type.TypeAs().WriteScope(ref row, ref valueScope, TypeArgumentList.Empty, out RowCursor mmmScope)); + + // Invalid because is a tuple type but with the wrong parameters. + ResultAssert.TypeConstraint( + mmmType.Type.TypeAs() + .WriteScope( + ref row, + ref valueScope, + new TypeArgumentList( + new[] + { + new TypeArgument(LayoutType.Boolean), + new TypeArgument(LayoutType.Int64), + }), + out mmmScope)); + + // Invalid because is a tuple type but with the wrong arity. + ResultAssert.TypeConstraint( + mmmType.Type.TypeAs() + .WriteScope( + ref row, + ref valueScope, + new TypeArgumentList( + new[] + { + new TypeArgument(LayoutType.Utf8), + }), + out mmmScope)); + + ResultAssert.IsSuccess( + mmmType.Type.TypeAs().WriteScope(ref row, ref valueScope, mmmType.TypeArgs, out mmmScope)); + ResultAssert.TypeConstraint(LayoutType.Binary.WriteSparse(ref row, ref valueScope, new byte[] { 1, 2, 3 })); + ResultAssert.IsSuccess(mmmType.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref mmmScope, 1L)); + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess(mmmType.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref mmmScope, 2L)); + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess(mmmType.TypeArgs[2].Type.TypeAs().WriteSparse(ref row, ref mmmScope, 3L)); + } + + [TestMethod] + [Owner("jthunter")] + public void VerifyTypeConstraintsCoordCounter() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.countersLayout, this.countersResolver); + + PerfCounter c1 = new PerfCounter() + { + Name = "RowInserts", + Coord = Tuple.Create("units", new Coord { Lat = 12L, Lng = 40L }), + }; + + this.WriteCounter(ref row, ref RowCursor.Create(ref row, out RowCursor _), c1); + + Assert.IsTrue(this.countersLayout.TryFind("coord", out LayoutColumn c)); + RowCursor.Create(ref row, out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + ResultAssert.TypeConstraint(LayoutType.DateTime.WriteSparse(ref row, ref valueScope, DateTime.Now)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, "mins")); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.TypeConstraint(LayoutType.Int8.WriteSparse(ref row, ref valueScope, 42)); + + TypeArgument coordType = c.TypeArgs[1]; + + // Invalid because is a UDT but the wrong type. + ResultAssert.TypeConstraint( + coordType.Type.TypeAs() + .WriteScope( + ref row, + ref valueScope, + new TypeArgumentList(this.countersLayout.SchemaId), + out RowCursor _)); + } + + [TestMethod] + [Owner("jthunter")] + public void DownwardDelegateWriteScope() + { + RowBuffer row = new RowBuffer(TupleUnitTests.InitialRowSize); + Layout layout = this.countersResolver.Resolve(this.counterSchema.Schemas.Find(x => x.Name == "CounterSet").SchemaId); + row.InitLayout(HybridRowVersion.V1, layout, this.countersResolver); + + Assert.IsTrue(layout.TryFind("history", out LayoutColumn col)); + Assert.IsTrue(layout.Tokenizer.TryFindToken(col.Path, out StringToken historyToken)); + RowCursor.Create(ref row, out RowCursor history).Find(ref row, historyToken); + int ctx = 1; // ignored + ResultAssert.IsSuccess( + LayoutType.TypedArray.WriteScope( + ref row, + ref history, + col.TypeArgs, + ctx, + (ref RowBuffer row2, ref RowCursor arrCur, int ctx2) => + { + for (int i = 0; i < 5; i++) + { + ResultAssert.IsSuccess( + LayoutType.UDT.WriteScope( + ref row2, + ref arrCur, + arrCur.ScopeTypeArgs[0].TypeArgs, + i, + (ref RowBuffer row3, ref RowCursor udtCur, int ctx3) => + { + Assert.IsTrue(udtCur.Layout.TryFind("minmeanmax", out LayoutColumn col3)); + ResultAssert.IsSuccess(LayoutType.TypedTuple.WriteScope( + ref row3, + ref udtCur.Find(ref row3, col3.Path), + col3.TypeArgs, + ctx3, + (ref RowBuffer row4, ref RowCursor tupCur, int ctx4) => + { + if (ctx4 > 0) + { + ResultAssert.IsSuccess(LayoutType.Utf8.WriteSparse(ref row4, ref tupCur, "abc")); + } + + if (ctx4 > 1) + { + Assert.IsTrue(tupCur.MoveNext(ref row4)); + ResultAssert.IsSuccess( + LayoutType.TypedTuple.WriteScope( + ref row4, + ref tupCur, + tupCur.ScopeTypeArgs[1].TypeArgs, + ctx4, + (ref RowBuffer row5, ref RowCursor tupCur2, int ctx5) => + { + if (ctx5 > 1) + { + ResultAssert.IsSuccess(LayoutType.Int64.WriteSparse(ref row5, ref tupCur2, ctx5)); + } + + if (ctx5 > 2) + { + Assert.IsTrue(tupCur2.MoveNext(ref row5)); + ResultAssert.IsSuccess(LayoutType.Int64.WriteSparse(ref row5, ref tupCur2, ctx5)); + } + + if (ctx5 > 3) + { + Assert.IsTrue(tupCur2.MoveNext(ref row5)); + ResultAssert.IsSuccess(LayoutType.Int64.WriteSparse(ref row5, ref tupCur2, ctx5)); + } + + return Result.Success; + })); + } + + return Result.Success; + })); + + return Result.Success; + })); + + Assert.IsFalse(arrCur.MoveNext(ref row2)); + } + + return Result.Success; + })); + } + + private void WriteCounter(ref RowBuffer row, ref RowCursor root, PerfCounter pc) + { + Assert.IsTrue(this.countersLayout.TryFind("name", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, pc.Name)); + + if (pc.Value != null) + { + Assert.IsTrue(this.countersLayout.TryFind("value", out c)); + root.Clone(out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, pc.Value.Item1)); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref valueScope, pc.Value.Item2)); + } + + if (pc.MinMaxValue != null) + { + // ReSharper disable once StringLiteralTypo + Assert.IsTrue(this.countersLayout.TryFind("minmeanmax", out c)); + root.Clone(out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, pc.MinMaxValue.Item1)); + + Assert.IsTrue(valueScope.MoveNext(ref row)); + TypeArgument mmmType = c.TypeArgs[1]; + ResultAssert.IsSuccess( + mmmType.Type.TypeAs().WriteScope(ref row, ref valueScope, mmmType.TypeArgs, out RowCursor mmmScope)); + + ResultAssert.IsSuccess( + mmmType.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref mmmScope, pc.MinMaxValue.Item2.Item1)); + + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + mmmType.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref mmmScope, pc.MinMaxValue.Item2.Item2)); + + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + mmmType.TypeArgs[2].Type.TypeAs().WriteSparse(ref row, ref mmmScope, pc.MinMaxValue.Item2.Item3)); + } + + if (pc.Coord != null) + { + Assert.IsTrue(this.countersLayout.TryFind("coord", out c)); + root.Clone(out RowCursor valueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref valueScope, c.TypeArgs, out valueScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref valueScope, pc.Coord.Item1)); + + Assert.IsTrue(valueScope.MoveNext(ref row)); + TypeArgument mmmType = c.TypeArgs[1]; + ResultAssert.IsSuccess( + mmmType.Type.TypeAs().WriteScope(ref row, ref valueScope, mmmType.TypeArgs, out RowCursor coordScope)); + TupleUnitTests.WriteCoord(ref row, ref coordScope, mmmType.TypeArgs, pc.Coord.Item2); + } + } + + private PerfCounter ReadCounter(ref RowBuffer row, ref RowCursor root) + { + PerfCounter pc = new PerfCounter(); + Assert.IsTrue(this.countersLayout.TryFind("name", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out pc.Name)); + + Assert.IsTrue(this.countersLayout.TryFind("value", out c)); + Assert.IsTrue(c.Type.Immutable); + root.Clone(out RowCursor valueScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref valueScope, out valueScope) == Result.Success) + { + Assert.IsTrue(valueScope.Immutable); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref valueScope, out string units)); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[1].Type.TypeAs().ReadSparse(ref row, ref valueScope, out long metric)); + pc.Value = Tuple.Create(units, metric); + } + + // ReSharper disable once StringLiteralTypo + Assert.IsTrue(this.countersLayout.TryFind("minmeanmax", out c)); + Assert.IsTrue(c.Type.Immutable); + root.Clone(out valueScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref valueScope, out valueScope) == Result.Success) + { + Assert.IsTrue(valueScope.Immutable); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref valueScope, out string units)); + + Assert.IsTrue(valueScope.MoveNext(ref row)); + TypeArgument mmmType = c.TypeArgs[1]; + ResultAssert.IsSuccess( + mmmType.Type.TypeAs().ReadScope(ref row, ref valueScope, out RowCursor mmmScope)); + + Assert.IsTrue(mmmScope.Immutable); + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess(mmmType.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref mmmScope, out long min)); + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess(mmmType.TypeArgs[1].Type.TypeAs().ReadSparse(ref row, ref mmmScope, out long mean)); + Assert.IsTrue(mmmScope.MoveNext(ref row)); + ResultAssert.IsSuccess(mmmType.TypeArgs[2].Type.TypeAs().ReadSparse(ref row, ref mmmScope, out long max)); + + pc.MinMaxValue = Tuple.Create(units, Tuple.Create(min, mean, max)); + } + + Assert.IsTrue(this.countersLayout.TryFind("coord", out c)); + Assert.IsTrue(c.Type.Immutable); + root.Clone(out valueScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref valueScope, out valueScope) == Result.Success) + { + Assert.IsTrue(valueScope.Immutable); + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref valueScope, out string units)); + + Assert.IsTrue(valueScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + c.TypeArgs[1].Type.TypeAs().ReadScope(ref row, ref valueScope, out RowCursor coordScope)); + pc.Coord = Tuple.Create(units, TupleUnitTests.ReadCoord(ref row, ref coordScope)); + } + + return pc; + } + + private static void WriteCoord(ref RowBuffer row, ref RowCursor coordScope, TypeArgumentList typeArgs, Coord cd) + { + Layout coordLayout = row.Resolver.Resolve(typeArgs.SchemaId); + Assert.IsTrue(coordLayout.TryFind("lat", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref coordScope, c, cd.Lat)); + Assert.IsTrue(coordLayout.TryFind("lng", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref coordScope, c, cd.Lng)); + } + + private static Coord ReadCoord(ref RowBuffer row, ref RowCursor coordScope) + { + Layout coordLayout = coordScope.Layout; + Coord cd = new Coord(); + Assert.IsTrue(coordLayout.TryFind("lat", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref coordScope, c, out cd.Lat)); + Assert.IsTrue(coordLayout.TryFind("lng", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref coordScope, c, out cd.Lng)); + + return cd; + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class PerfCounter + { + public string Name; + public Tuple Value; + public Tuple> MinMaxValue; + + // ReSharper disable once MemberHidesStaticFromOuterClass + public Tuple Coord; + + // ReSharper disable once MemberCanBePrivate.Local + public bool Equals(PerfCounter other) + { + return string.Equals(this.Name, other.Name) && + object.Equals(this.Value, other.Value) && + object.Equals(this.MinMaxValue, other.MinMaxValue) && + object.Equals(this.Coord, other.Coord); + } + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is PerfCounter counter && this.Equals(counter); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = this.Name?.GetHashCode() ?? 0; + hashCode = (hashCode * 397) ^ (this.Value?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.MinMaxValue?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Coord?.GetHashCode() ?? 0); + return hashCode; + } + } + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class Coord + { + public long Lat; + public long Lng; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Coord coord && this.Equals(coord); + } + + public override int GetHashCode() + { + unchecked + { + return (this.Lat.GetHashCode() * 397) ^ this.Lng.GetHashCode(); + } + } + + private bool Equals(Coord other) + { + return this.Lat == other.Lat && this.Lng == other.Lng; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TypedArrayUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/TypedArrayUnitTests.cs new file mode 100644 index 0000000..5e0cd97 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TypedArrayUnitTests.cs @@ -0,0 +1,413 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +// ReSharper disable StringLiteralTypo +// ReSharper disable IdentifierTypo +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Linq; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(TypedArrayUnitTests.SchemaFile, "TestData")] + public sealed class TypedArrayUnitTests + { + private const string SchemaFile = @"TestData\TagSchema.json"; + private const int InitialRowSize = 2 * 1024 * 1024; + + private Namespace counterSchema; + private LayoutResolver resolver; + private Layout layout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(TypedArrayUnitTests.SchemaFile); + this.counterSchema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.counterSchema); + this.layout = this.resolver.Resolve(this.counterSchema.Schemas.Find(x => x.Name == "Tagged").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateTags() + { + RowBuffer row = new RowBuffer(TypedArrayUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + Tagged t1 = new Tagged + { + Title = "Thriller", + Tags = new List { "classic", "Post-disco", "funk" }, + Options = new List { 8, null, 9 }, + Ratings = new List> + { + new List { 1.2, 3.0 }, + new List { 4.1, 5.7 }, + new List { 7.3, 8.12, 9.14 }, + }, + Similars = new List + { + new SimilarMatch { Thumbprint = "TRABACN128F425B784", Score = 0.87173699999999998 }, + new SimilarMatch { Thumbprint = "TRJYGLF12903CB4952", Score = 0.75105200000000005 }, + new SimilarMatch { Thumbprint = "TRWJMMB128F429D550", Score = 0.50866100000000003 }, + }, + Priority = new List> + { + Tuple.Create("80's", 100L), + Tuple.Create("classics", 100L), + Tuple.Create("pop", 50L), + }, + }; + + this.WriteTagged(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + Tagged t2 = this.ReadTagged(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(t1, t2); + } + + private void WriteTagged(ref RowBuffer row, ref RowCursor root, Tagged value) + { + Assert.IsTrue(this.layout.TryFind("title", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref root, c, value.Title)); + + if (value.Tags != null) + { + Assert.IsTrue(this.layout.TryFind("tags", out c)); + root.Clone(out RowCursor tagsScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref tagsScope, c.TypeArgs, out tagsScope)); + foreach (string item in value.Tags) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tagsScope, item)); + Assert.IsFalse(tagsScope.MoveNext(ref row)); + } + } + + if (value.Options != null) + { + Assert.IsTrue(this.layout.TryFind("options", out c)); + root.Clone(out RowCursor optionsScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref optionsScope, c.TypeArgs, out optionsScope)); + foreach (int? item in value.Options) + { + TypeArgument itemType = c.TypeArgs[0]; + ResultAssert.IsSuccess( + itemType.Type.TypeAs() + .WriteScope(ref row, ref optionsScope, itemType.TypeArgs, item.HasValue, out RowCursor nullableScope)); + + if (item.HasValue) + { + ResultAssert.IsSuccess( + itemType.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref nullableScope, item.Value)); + } + + Assert.IsFalse(optionsScope.MoveNext(ref row, ref nullableScope)); + } + } + + if (value.Ratings != null) + { + Assert.IsTrue(this.layout.TryFind("ratings", out c)); + root.Clone(out RowCursor ratingsScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref ratingsScope, c.TypeArgs, out ratingsScope)); + foreach (List item in value.Ratings) + { + Assert.IsTrue(item != null); + TypeArgument innerType = c.TypeArgs[0]; + LayoutTypedArray innerLayout = innerType.Type.TypeAs(); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref ratingsScope, innerType.TypeArgs, out RowCursor innerScope)); + foreach (double innerItem in item) + { + LayoutFloat64 itemLayout = innerType.TypeArgs[0].Type.TypeAs(); + ResultAssert.IsSuccess(itemLayout.WriteSparse(ref row, ref innerScope, innerItem)); + Assert.IsFalse(innerScope.MoveNext(ref row)); + } + + Assert.IsFalse(ratingsScope.MoveNext(ref row, ref innerScope)); + } + } + + if (value.Similars != null) + { + Assert.IsTrue(this.layout.TryFind("similars", out c)); + root.Clone(out RowCursor similarsScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref similarsScope, c.TypeArgs, out similarsScope)); + foreach (SimilarMatch item in value.Similars) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutUDT innerLayout = innerType.Type.TypeAs(); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref similarsScope, innerType.TypeArgs, out RowCursor matchScope)); + TypedArrayUnitTests.WriteSimilarMatch(ref row, ref matchScope, innerType.TypeArgs, item); + + Assert.IsFalse(similarsScope.MoveNext(ref row, ref matchScope)); + } + } + + if (value.Priority != null) + { + Assert.IsTrue(this.layout.TryFind("priority", out c)); + root.Clone(out RowCursor priorityScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref priorityScope, c.TypeArgs, out priorityScope)); + foreach (Tuple item in value.Priority) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutIndexedScope innerLayout = innerType.Type.TypeAs(); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref priorityScope, innerType.TypeArgs, out RowCursor tupleScope)); + ResultAssert.IsSuccess(innerType.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tupleScope, item.Item1)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(innerType.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref tupleScope, item.Item2)); + + Assert.IsFalse(priorityScope.MoveNext(ref row, ref tupleScope)); + } + } + } + + private Tagged ReadTagged(ref RowBuffer row, ref RowCursor root) + { + Tagged value = new Tagged(); + Assert.IsTrue(this.layout.TryFind("title", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref root, c, out value.Title)); + + Assert.IsTrue(this.layout.TryFind("tags", out c)); + root.Clone(out RowCursor tagsScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref tagsScope, out tagsScope) == Result.Success) + { + value.Tags = new List(); + while (tagsScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tagsScope, out string item)); + value.Tags.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("options", out c)); + root.Clone(out RowCursor optionsScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref optionsScope, out optionsScope) == Result.Success) + { + value.Options = new List(); + while (optionsScope.MoveNext(ref row)) + { + TypeArgument itemType = c.TypeArgs[0]; + ResultAssert.IsSuccess( + itemType.Type.TypeAs() + .ReadScope(ref row, ref optionsScope, out RowCursor nullableScope)); + + if (nullableScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(LayoutNullable.HasValue(ref row, ref nullableScope)); + + ResultAssert.IsSuccess( + itemType.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref nullableScope, out int itemValue)); + + value.Options.Add(itemValue); + } + else + { + ResultAssert.NotFound(LayoutNullable.HasValue(ref row, ref nullableScope)); + + value.Options.Add(null); + } + } + } + + Assert.IsTrue(this.layout.TryFind("ratings", out c)); + root.Clone(out RowCursor ratingsScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref ratingsScope, out ratingsScope) == Result.Success) + { + value.Ratings = new List>(); + TypeArgument innerType = c.TypeArgs[0]; + LayoutTypedArray innerLayout = innerType.Type.TypeAs(); + RowCursor innerScope = default; + while (ratingsScope.MoveNext(ref row, ref innerScope)) + { + List item = new List(); + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref ratingsScope, out innerScope)); + while (innerScope.MoveNext(ref row)) + { + LayoutFloat64 itemLayout = innerType.TypeArgs[0].Type.TypeAs(); + ResultAssert.IsSuccess(itemLayout.ReadSparse(ref row, ref innerScope, out double innerItem)); + item.Add(innerItem); + } + + value.Ratings.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("similars", out c)); + root.Clone(out RowCursor similarsScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref similarsScope, out similarsScope) == Result.Success) + { + value.Similars = new List(); + while (similarsScope.MoveNext(ref row)) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutUDT innerLayout = innerType.Type.TypeAs(); + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref similarsScope, out RowCursor matchScope)); + SimilarMatch item = TypedArrayUnitTests.ReadSimilarMatch(ref row, ref matchScope); + value.Similars.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("priority", out c)); + root.Clone(out RowCursor priorityScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref priorityScope, out priorityScope) == Result.Success) + { + value.Priority = new List>(); + RowCursor tupleScope = default; + while (priorityScope.MoveNext(ref row, ref tupleScope)) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutIndexedScope innerLayout = innerType.Type.TypeAs(); + + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref priorityScope, out tupleScope)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + innerType.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tupleScope, out string item1)); + + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess( + innerType.TypeArgs[1].Type.TypeAs().ReadSparse(ref row, ref tupleScope, out long item2)); + + value.Priority.Add(Tuple.Create(item1, item2)); + } + } + + return value; + } + + private static void WriteSimilarMatch(ref RowBuffer row, ref RowCursor matchScope, TypeArgumentList typeArgs, SimilarMatch m) + { + Layout matchLayout = row.Resolver.Resolve(typeArgs.SchemaId); + Assert.IsTrue(matchLayout.TryFind("thumbprint", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref matchScope, c, m.Thumbprint)); + Assert.IsTrue(matchLayout.TryFind("score", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref matchScope, c, m.Score)); + } + + private static SimilarMatch ReadSimilarMatch(ref RowBuffer row, ref RowCursor matchScope) + { + Layout matchLayout = matchScope.Layout; + SimilarMatch m = new SimilarMatch(); + Assert.IsTrue(matchLayout.TryFind("thumbprint", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref matchScope, c, out m.Thumbprint)); + Assert.IsTrue(matchLayout.TryFind("score", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref matchScope, c, out m.Score)); + return m; + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class Tagged + { + public string Title; + public List Tags; + public List Options; + public List> Ratings; + public List Similars; + public List> Priority; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Tagged tagged && this.Equals(tagged); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = this.Title?.GetHashCode() ?? 0; + hashCode = (hashCode * 397) ^ (this.Tags?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Options?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Ratings?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Similars?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Priority?.GetHashCode() ?? 0); + return hashCode; + } + } + + private static bool NestedSequenceEquals(List> left, List> right) + { + if (left.Count != right.Count) + { + return false; + } + + for (int i = 0; i < left.Count; i++) + { + if (!left[i].SequenceEqual(right[i])) + { + return false; + } + } + + return true; + } + + private bool Equals(Tagged other) + { + return string.Equals(this.Title, other.Title) && + (object.ReferenceEquals(this.Tags, other.Tags) || + ((this.Tags != null) && (other.Tags != null) && this.Tags.SequenceEqual(other.Tags))) && + (object.ReferenceEquals(this.Options, other.Options) || + ((this.Options != null) && (other.Options != null) && this.Options.SequenceEqual(other.Options))) && + (object.ReferenceEquals(this.Ratings, other.Ratings) || + ((this.Ratings != null) && (other.Ratings != null) && Tagged.NestedSequenceEquals(this.Ratings, other.Ratings))) && + (object.ReferenceEquals(this.Similars, other.Similars) || + ((this.Similars != null) && (other.Similars != null) && this.Similars.SequenceEqual(other.Similars))) && + (object.ReferenceEquals(this.Priority, other.Priority) || + ((this.Priority != null) && (other.Priority != null) && this.Priority.SequenceEqual(other.Priority))); + } + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class SimilarMatch + { + public string Thumbprint; + public double Score; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is SimilarMatch match && this.Equals(match); + } + + public override int GetHashCode() + { + unchecked + { + return (this.Thumbprint.GetHashCode() * 397) ^ this.Score.GetHashCode(); + } + } + + private bool Equals(SimilarMatch other) + { + // ReSharper disable once CompareOfFloatsByEqualityOperator + return this.Thumbprint == other.Thumbprint && this.Score == other.Score; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TypedMapUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/TypedMapUnitTests.cs new file mode 100644 index 0000000..4332bec --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TypedMapUnitTests.cs @@ -0,0 +1,704 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(TypedMapUnitTests.SchemaFile, "TestData")] + public sealed class TypedMapUnitTests + { + private const string SchemaFile = @"TestData\MovieSchema.json"; + private const int InitialRowSize = 2 * 1024 * 1024; + + private Namespace counterSchema; + private LayoutResolver resolver; + private Layout layout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(TypedMapUnitTests.SchemaFile); + this.counterSchema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.counterSchema); + this.layout = this.resolver.Resolve(this.counterSchema.Schemas.Find(x => x.Name == "Movie").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateMovies() + { + RowBuffer row = new RowBuffer(TypedMapUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + // ReSharper disable StringLiteralTypo + Movie t1 = new Movie + { + Cast = new Dictionary { { "Mark", "Luke" }, { "Harrison", "Han" }, { "Carrie", "Leia" } }, + Stats = new Dictionary + { + { Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), 11000000.00 }, + { Guid.Parse("{7499C40E-7077-45C1-AE5F-3E384966B3B9}"), 1554475.00 }, + }, + Related = new Dictionary> + { + { "Mark", new Dictionary { { 103359, "Joker" }, { 131646, "Merlin" } } }, + { "Harrison", new Dictionary { { 0082971, "Indy" }, { 83658, "Deckard" } } }, + }, + Revenue = new Dictionary + { + { DateTime.Parse("05/25/1977"), new Earnings { Domestic = 307263857M, Worldwide = 100000M } }, + { DateTime.Parse("08/13/1982"), new Earnings { Domestic = 15476285M, Worldwide = 200000M } }, + { DateTime.Parse("01/31/1997"), new Earnings { Domestic = 138257865M, Worldwide = 300000M } }, + }, + }; + + // ReSharper restore StringLiteralTypo + this.WriteMovie(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + Movie t2 = this.ReadMovie(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(t1, t2); + } + + [TestMethod] + [Owner("jthunter")] + public void PreventUpdatesInNonUpdatableScope() + { + RowBuffer row = new RowBuffer(TypedMapUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + RowCursor root = RowCursor.Create(ref row); + + // Write a map and then try to write directly into it. + Assert.IsTrue(this.layout.TryFind("cast", out LayoutColumn c)); + root.Clone(out RowCursor mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref mapScope, c.TypeArgs, out mapScope)); + ResultAssert.InsufficientPermissions( + TypedMapUnitTests.WriteKeyValue(ref row, ref mapScope, c.TypeArgs, KeyValuePair.Create("Mark", "Joker"))); + root.Clone(out RowCursor tempCursor).Find(ref row, "cast.0"); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, KeyValuePair.Create("Mark", "Joker"))); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor)); + root.Clone(out tempCursor).Find(ref row, "cast.0"); + ResultAssert.NotFound(TypedMapUnitTests.ReadKeyValue(ref row, ref tempCursor, c.TypeArgs, out KeyValuePair _)); + + // Write a map of maps, successfully insert an empty map into it, and then try to write directly to the inner map. + Assert.IsTrue(this.layout.TryFind("related", out c)); + root.Clone(out mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref mapScope, c.TypeArgs, out mapScope)); + LayoutIndexedScope tupleLayout = c.TypeAs().FieldType(ref mapScope).TypeAs(); + root.Clone(out tempCursor).Find(ref row, "related.0"); + ResultAssert.IsSuccess(tupleLayout.WriteScope(ref row, ref tempCursor, c.TypeArgs, out RowCursor tupleScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tupleScope, "Mark")); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + TypeArgument valueType = c.TypeArgs[1]; + LayoutUniqueScope valueLayout = valueType.Type.TypeAs(); + ResultAssert.IsSuccess(valueLayout.WriteScope(ref row, ref tupleScope, valueType.TypeArgs, out RowCursor innerScope)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor)); + Assert.IsTrue(mapScope.MoveNext(ref row)); + ResultAssert.IsSuccess(tupleLayout.ReadScope(ref row, ref mapScope, out tupleScope)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + + // Skip key. + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(valueLayout.ReadScope(ref row, ref tupleScope, out innerScope)); + TypeArgument itemType = valueType.TypeArgs[0]; + Assert.IsFalse(innerScope.MoveNext(ref row)); + ResultAssert.InsufficientPermissions(itemType.Type.TypeAs().WriteSparse(ref row, ref innerScope, 1)); + ResultAssert.InsufficientPermissions(itemType.Type.TypeAs().DeleteSparse(ref row, ref innerScope)); + } + + [TestMethod] + [Owner("jthunter")] + public void PreventUniquenessViolations() + { + RowBuffer row = new RowBuffer(TypedMapUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + RowCursor root = RowCursor.Create(ref row); + + Movie t1 = new Movie + { + Cast = new Dictionary { { "Mark", "Luke" } }, + Stats = new Dictionary + { + { Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), 11000000.00 }, + }, + Related = new Dictionary> + { + { "Mark", new Dictionary { { 103359, "Joker" } } }, + }, + Revenue = new Dictionary + { + { DateTime.Parse("05/25/1977"), new Earnings { Domestic = 307263857M, Worldwide = 100000M } }, + }, + }; + + RowCursor rc1 = RowCursor.Create(ref row); + this.WriteMovie(ref row, ref rc1, t1); + + // Attempt to insert duplicate items in existing sets. + Assert.IsTrue(this.layout.TryFind("cast", out LayoutColumn c)); + root.Clone(out RowCursor mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref mapScope, out mapScope)); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, KeyValuePair.Create("Mark", "Luke"))); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.Insert)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.NotFound(TypedMapUnitTests.ReadKeyValue(ref row, ref tempCursor, c.TypeArgs, out KeyValuePair _)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess( + TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, KeyValuePair.Create("Mark", "Joker"))); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.Insert)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.NotFound(TypedMapUnitTests.ReadKeyValue(ref row, ref tempCursor, c.TypeArgs, out KeyValuePair _)); + + Assert.IsTrue(this.layout.TryFind("stats", out c)); + root.Clone(out mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref mapScope, out mapScope)); + KeyValuePair pair = KeyValuePair.Create(Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), 11000000.00); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, pair)); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.Insert)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.NotFound(TypedMapUnitTests.ReadKeyValue(ref row, ref tempCursor, c.TypeArgs, out pair)); + } + + [TestMethod] + [Owner("jthunter")] + public void FindInMap() + { + RowBuffer row = new RowBuffer(TypedMapUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + RowCursor root = RowCursor.Create(ref row); + + Movie t1 = new Movie + { + Cast = new Dictionary { { "Mark", "Luke" }, { "Harrison", "Han" }, { "Carrie", "Leia" } }, + }; + RowCursor rc1 = RowCursor.Create(ref row); + this.WriteMovie(ref row, ref rc1, t1); + + // Attempt to find each item in turn. + Assert.IsTrue(this.layout.TryFind("cast", out LayoutColumn c)); + root.Clone(out RowCursor mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref mapScope, out mapScope)); + foreach (string key in t1.Cast.Keys) + { + KeyValuePair pair = new KeyValuePair(key, "map lookup matches only on key"); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, pair)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref mapScope, ref tempCursor, out RowCursor findScope)); + ResultAssert.IsSuccess( + TypedMapUnitTests.ReadKeyValue(ref row, ref findScope, c.TypeArgs, out KeyValuePair foundPair)); + Assert.AreEqual(key, foundPair.Key, $"Failed to find t1.Cast[{key}]"); + } + } + + [TestMethod] + [Owner("jthunter")] + public void UpdateInMap() + { + RowBuffer row = new RowBuffer(TypedMapUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + RowCursor root = RowCursor.Create(ref row); + + List expected = new List { "Mark", "Harrison", "Carrie", }; + + foreach (IEnumerable permutation in expected.Permute()) + { + Movie t1 = new Movie + { + Cast = new Dictionary { { "Mark", "Luke" }, { "Harrison", "Han" }, { "Carrie", "Leia" } }, + }; + this.WriteMovie(ref row, ref root.Clone(out RowCursor _), t1); + + // Attempt to find each item in turn and then delete it. + Assert.IsTrue(this.layout.TryFind("cast", out LayoutColumn c)); + root.Clone(out RowCursor mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref mapScope, out mapScope)); + foreach (string key in permutation) + { + // Verify it is already there. + KeyValuePair pair = new KeyValuePair(key, "map lookup matches only on key"); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, pair)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref mapScope, ref tempCursor, out RowCursor findScope)); + + // Insert it again with update. + KeyValuePair updatePair = new KeyValuePair(key, "update value"); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, updatePair)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.Update)); + + // Verify that the value was updated. + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, pair)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref mapScope, ref tempCursor, out findScope)); + ResultAssert.IsSuccess( + TypedMapUnitTests.ReadKeyValue(ref row, ref findScope, c.TypeArgs, out KeyValuePair foundPair)); + Assert.AreEqual(key, foundPair.Key); + Assert.AreEqual(updatePair.Value, foundPair.Value); + + // Insert it again with upsert. + updatePair = new KeyValuePair(key, "upsert value"); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, updatePair)); + + // ReSharper disable once RedundantArgumentDefaultValue + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.Upsert)); + + // Verify that the value was upserted. + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, pair)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref mapScope, ref tempCursor, out findScope)); + ResultAssert.IsSuccess(TypedMapUnitTests.ReadKeyValue(ref row, ref findScope, c.TypeArgs, out foundPair)); + Assert.AreEqual(key, foundPair.Key); + Assert.AreEqual(updatePair.Value, foundPair.Value); + + // Insert it again with insert (fail: exists). + updatePair = new KeyValuePair(key, "insert value"); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, updatePair)); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.Insert)); + + // Insert it again with insert at (fail: disallowed). + updatePair = new KeyValuePair(key, "insertAt value"); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, updatePair)); + ResultAssert.TypeConstraint( + c.TypeAs().MoveField(ref row, ref mapScope, ref tempCursor, UpdateOptions.InsertAt)); + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void FindAndDelete() + { + RowBuffer row = new RowBuffer(TypedMapUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + RowCursor root = RowCursor.Create(ref row); + + List expected = new List { "Mark", "Harrison", "Carrie", }; + + foreach (IEnumerable permutation in expected.Permute()) + { + Movie t1 = new Movie + { + Cast = new Dictionary { { "Mark", "Luke" }, { "Harrison", "Han" }, { "Carrie", "Leia" } }, + }; + this.WriteMovie(ref row, ref root.Clone(out RowCursor _), t1); + + // Attempt to find each item in turn and then delete it. + Assert.IsTrue(this.layout.TryFind("cast", out LayoutColumn c)); + root.Clone(out RowCursor mapScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref mapScope, out mapScope)); + foreach (string key in permutation) + { + KeyValuePair pair = new KeyValuePair(key, "map lookup matches only on key"); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, pair)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref mapScope, ref tempCursor, out RowCursor findScope)); + TypeArgument tupleType = c.TypeAs().FieldType(ref mapScope); + ResultAssert.IsSuccess(tupleType.TypeAs().DeleteScope(ref row, ref findScope)); + } + } + } + + private static Result WriteKeyValue( + ref RowBuffer row, + ref RowCursor scope, + TypeArgumentList typeArgs, + KeyValuePair pair) + { + LayoutIndexedScope tupleLayout = LayoutType.TypedTuple; + Result r = tupleLayout.WriteScope(ref row, ref scope, typeArgs, out RowCursor tupleScope); + if (r != Result.Success) + { + return r; + } + + r = typeArgs[0].Type.TypeAs>().WriteSparse(ref row, ref tupleScope, pair.Key); + if (r != Result.Success) + { + return r; + } + + tupleScope.MoveNext(ref row); + r = typeArgs[1].Type.TypeAs>().WriteSparse(ref row, ref tupleScope, pair.Value); + if (r != Result.Success) + { + return r; + } + + return Result.Success; + } + + private static Result ReadKeyValue( + ref RowBuffer row, + ref RowCursor scope, + TypeArgumentList typeArgs, + out KeyValuePair pair) + { + pair = default; + LayoutIndexedScope tupleLayout = LayoutType.TypedTuple; + Result r = tupleLayout.ReadScope(ref row, ref scope, out RowCursor tupleScope); + if (r != Result.Success) + { + return r; + } + + tupleScope.MoveNext(ref row); + r = typeArgs[0].Type.TypeAs>().ReadSparse(ref row, ref tupleScope, out TKey key); + if (r != Result.Success) + { + return r; + } + + tupleScope.MoveNext(ref row); + r = typeArgs[1].Type.TypeAs>().ReadSparse(ref row, ref tupleScope, out TValue value); + if (r != Result.Success) + { + return r; + } + + pair = new KeyValuePair(key, value); + return Result.Success; + } + + private void WriteMovie(ref RowBuffer row, ref RowCursor root, Movie value) + { + LayoutColumn c; + + if (value.Cast != null) + { + Assert.IsTrue(this.layout.TryFind("cast", out c)); + root.Clone(out RowCursor castScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref castScope, c.TypeArgs, out castScope)); + foreach (KeyValuePair item in value.Cast) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, item)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref castScope, ref tempCursor)); + } + } + + if (value.Stats != null) + { + Assert.IsTrue(this.layout.TryFind("stats", out c)); + root.Clone(out RowCursor statsScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref statsScope, c.TypeArgs, out statsScope)); + foreach (KeyValuePair item in value.Stats) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor, c.TypeArgs, item)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref statsScope, ref tempCursor)); + } + } + + if (value.Related != null) + { + Assert.IsTrue(this.layout.TryFind("related", out c)); + root.Clone(out RowCursor relatedScoped).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref relatedScoped, c.TypeArgs, out relatedScoped)); + foreach (KeyValuePair> item in value.Related) + { + Assert.IsTrue(item.Value != null); + + LayoutIndexedScope tupleLayout = LayoutType.TypedTuple; + root.Clone(out RowCursor tempCursor1).Find(ref row, "related.0"); + ResultAssert.IsSuccess(tupleLayout.WriteScope(ref row, ref tempCursor1, c.TypeArgs, out RowCursor tupleScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tupleScope, item.Key)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + TypeArgument valueType = c.TypeArgs[1]; + LayoutUniqueScope valueLayout = valueType.Type.TypeAs(); + ResultAssert.IsSuccess(valueLayout.WriteScope(ref row, ref tupleScope, valueType.TypeArgs, out RowCursor innerScope)); + foreach (KeyValuePair innerItem in item.Value) + { + root.Clone(out RowCursor tempCursor2).Find(ref row, "related.0.0"); + ResultAssert.IsSuccess(TypedMapUnitTests.WriteKeyValue(ref row, ref tempCursor2, valueType.TypeArgs, innerItem)); + ResultAssert.IsSuccess(valueLayout.MoveField(ref row, ref innerScope, ref tempCursor2)); + } + + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref relatedScoped, ref tempCursor1)); + } + } + + if (value.Revenue != null) + { + Assert.IsTrue(this.layout.TryFind("revenue", out c)); + root.Clone(out RowCursor revenueScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref revenueScope, c.TypeArgs, out revenueScope)); + foreach (KeyValuePair item in value.Revenue) + { + Assert.IsTrue(item.Value != null); + + LayoutIndexedScope tupleLayout = LayoutType.TypedTuple; + root.Clone(out RowCursor tempCursor1).Find(ref row, "revenue.0"); + ResultAssert.IsSuccess(tupleLayout.WriteScope(ref row, ref tempCursor1, c.TypeArgs, out RowCursor tupleScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tupleScope, item.Key)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + TypeArgument valueType = c.TypeArgs[1]; + LayoutUDT valueLayout = valueType.Type.TypeAs(); + ResultAssert.IsSuccess(valueLayout.WriteScope(ref row, ref tupleScope, valueType.TypeArgs, out RowCursor itemScope)); + TypedMapUnitTests.WriteEarnings(ref row, ref itemScope, valueType.TypeArgs, item.Value); + + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref revenueScope, ref tempCursor1)); + } + } + } + + private Movie ReadMovie(ref RowBuffer row, ref RowCursor root) + { + Movie value = new Movie(); + + Assert.IsTrue(this.layout.TryFind("cast", out LayoutColumn c)); + root.Clone(out RowCursor castScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref castScope, out castScope) == Result.Success) + { + value.Cast = new Dictionary(); + while (castScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(TypedMapUnitTests.ReadKeyValue(ref row, ref castScope, c.TypeArgs, out KeyValuePair item)); + value.Cast.Add(item.Key, item.Value); + } + } + + Assert.IsTrue(this.layout.TryFind("stats", out c)); + root.Clone(out RowCursor statsScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref statsScope, out statsScope) == Result.Success) + { + value.Stats = new Dictionary(); + while (statsScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(TypedMapUnitTests.ReadKeyValue(ref row, ref statsScope, c.TypeArgs, out KeyValuePair item)); + value.Stats.Add(item.Key, item.Value); + } + } + + Assert.IsTrue(this.layout.TryFind("related", out c)); + root.Clone(out RowCursor relatedScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref relatedScope, out relatedScope) == Result.Success) + { + value.Related = new Dictionary>(); + TypeArgument keyType = c.TypeArgs[0]; + TypeArgument valueType = c.TypeArgs[1]; + LayoutUtf8 keyLayout = keyType.Type.TypeAs(); + LayoutUniqueScope valueLayout = valueType.Type.TypeAs(); + while (relatedScope.MoveNext(ref row)) + { + LayoutIndexedScope tupleLayout = LayoutType.TypedTuple; + ResultAssert.IsSuccess(tupleLayout.ReadScope(ref row, ref relatedScope, out RowCursor tupleScope)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(keyLayout.ReadSparse(ref row, ref tupleScope, out string itemKey)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(valueLayout.ReadScope(ref row, ref tupleScope, out RowCursor itemValueScope)); + Dictionary itemValue = new Dictionary(); + while (itemValueScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess( + TypedMapUnitTests.ReadKeyValue( + ref row, + ref itemValueScope, + valueType.TypeArgs, + out KeyValuePair innerItem)); + itemValue.Add(innerItem.Key, innerItem.Value); + } + + value.Related.Add(itemKey, itemValue); + } + } + + Assert.IsTrue(this.layout.TryFind("revenue", out c)); + root.Clone(out RowCursor revenueScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref revenueScope, out revenueScope) == Result.Success) + { + value.Revenue = new Dictionary(); + TypeArgument keyType = c.TypeArgs[0]; + TypeArgument valueType = c.TypeArgs[1]; + LayoutDateTime keyLayout = keyType.Type.TypeAs(); + LayoutUDT valueLayout = valueType.Type.TypeAs(); + while (revenueScope.MoveNext(ref row)) + { + LayoutIndexedScope tupleLayout = LayoutType.TypedTuple; + ResultAssert.IsSuccess(tupleLayout.ReadScope(ref row, ref revenueScope, out RowCursor tupleScope)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(keyLayout.ReadSparse(ref row, ref tupleScope, out DateTime itemKey)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(valueLayout.ReadScope(ref row, ref tupleScope, out RowCursor itemValueScope)); + Earnings itemValue = TypedMapUnitTests.ReadEarnings(ref row, ref itemValueScope); + + value.Revenue.Add(itemKey, itemValue); + } + } + + return value; + } + + private static void WriteEarnings(ref RowBuffer row, ref RowCursor udtScope, TypeArgumentList typeArgs, Earnings m) + { + Layout udt = row.Resolver.Resolve(typeArgs.SchemaId); + Assert.IsTrue(udt.TryFind("domestic", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref udtScope, c, m.Domestic)); + Assert.IsTrue(udt.TryFind("worldwide", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref udtScope, c, m.Worldwide)); + } + + private static Earnings ReadEarnings(ref RowBuffer row, ref RowCursor udtScope) + { + Layout udt = udtScope.Layout; + Earnings m = new Earnings(); + Assert.IsTrue(udt.TryFind("domestic", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref udtScope, c, out m.Domestic)); + Assert.IsTrue(udt.TryFind("worldwide", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref udtScope, c, out m.Worldwide)); + return m; + } + + private static class KeyValuePair + { + public static KeyValuePair Create(TKey key, TValue value) + { + return new KeyValuePair(key, value); + } + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class Movie + { + public Dictionary Cast; + public Dictionary Stats; + public Dictionary> Related; + public Dictionary Revenue; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Movie movie && this.Equals(movie); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = 0; + hashCode = (hashCode * 397) ^ (this.Cast?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Stats?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Related?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Revenue?.GetHashCode() ?? 0); + return hashCode; + } + } + + private static bool NestedMapEquals( + Dictionary> left, + Dictionary> right) + { + if (left.Count != right.Count) + { + return false; + } + + foreach (KeyValuePair> item in left) + { + if (!right.TryGetValue(item.Key, out Dictionary value)) + { + return false; + } + + if (!Movie.MapEquals(item.Value, value)) + { + return false; + } + } + + return true; + } + + private static bool MapEquals(Dictionary left, Dictionary right) + { + if (left.Count != right.Count) + { + return false; + } + + foreach (KeyValuePair item in left) + { + if (!right.TryGetValue(item.Key, out TValue value)) + { + return false; + } + + if (!item.Value.Equals(value)) + { + return false; + } + } + + return true; + } + + private bool Equals(Movie other) + { + return (object.ReferenceEquals(this.Cast, other.Cast) || + ((this.Cast != null) && (other.Cast != null) && Movie.MapEquals(this.Cast, other.Cast))) && + (object.ReferenceEquals(this.Stats, other.Stats) || + ((this.Stats != null) && (other.Stats != null) && Movie.MapEquals(this.Stats, other.Stats))) && + (object.ReferenceEquals(this.Related, other.Related) || + ((this.Related != null) && (other.Related != null) && Movie.NestedMapEquals(this.Related, other.Related))) && + (object.ReferenceEquals(this.Revenue, other.Revenue) || + ((this.Revenue != null) && (other.Revenue != null) && Movie.MapEquals(this.Revenue, other.Revenue))); + } + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class Earnings + { + public decimal Domestic; + public decimal Worldwide; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Earnings earnings && this.Equals(earnings); + } + + public override int GetHashCode() + { + unchecked + { + return (this.Domestic.GetHashCode() * 397) ^ this.Worldwide.GetHashCode(); + } + } + + private bool Equals(Earnings other) + { + return this.Domestic == other.Domestic && this.Worldwide == other.Worldwide; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/TypedSetUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/TypedSetUnitTests.cs new file mode 100644 index 0000000..32c5209 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/TypedSetUnitTests.cs @@ -0,0 +1,877 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [DeploymentItem(TypedSetUnitTests.SchemaFile, "TestData")] + public sealed class TypedSetUnitTests + { + private const string SchemaFile = @"TestData\TodoSchema.json"; + private const int InitialRowSize = 2 * 1024 * 1024; + + private Namespace counterSchema; + private LayoutResolver resolver; + private Layout layout; + + [TestInitialize] + public void ParseNamespaceExample() + { + string json = File.ReadAllText(TypedSetUnitTests.SchemaFile); + this.counterSchema = Namespace.Parse(json); + this.resolver = new LayoutResolverNamespace(this.counterSchema); + this.layout = this.resolver.Resolve(this.counterSchema.Schemas.Find(x => x.Name == "Todo").SchemaId); + } + + [TestMethod] + [Owner("jthunter")] + public void CreateTodos() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + Todo t1 = new Todo + { + Attendees = new List { "jason", "janice", "joshua" }, + Projects = new List + { + Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), + Guid.Parse("{7499C40E-7077-45C1-AE5F-3E384966B3B9}"), + Guid.Parse("{B7BC39C2-1A2D-4EAF-8F33-ED976872D876}"), + Guid.Parse("{DEA71ABE-3041-4CAF-BBD9-1A46D10832A0}"), + }, + Checkboxes = new List { true, false }, + Prices = new List> + { + new List { 1.2F, 3.0F }, + new List { 4.1F, 5.7F }, + new List { 7.3F, 8.12F, 9.14F }, + }, + Nested = new List>> + { + new List> { new List { 1, 2 } }, + new List> { new List { 3, 4 } }, + new List> { new List { 5, 6 } }, + }, + Shopping = new List + { + new ShoppingItem { Label = "milk", Count = 1 }, + new ShoppingItem { Label = "broccoli", Count = 2 }, + new ShoppingItem { Label = "steak", Count = 6 }, + }, + Work = new List> + { + Tuple.Create(false, 10000UL), + Tuple.Create(true, 49053UL), + Tuple.Create(false, 53111UL), + }, + }; + + this.WriteTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + Todo t2 = this.ReadTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(t1, t2); + } + + [TestMethod] + [Owner("jthunter")] + public void PreventUpdatesInNonUpdatableScope() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + // Write a set and then try to write directly into it. + Assert.IsTrue(this.layout.TryFind("attendees", out LayoutColumn c)); + RowCursor.Create(ref row, out RowCursor setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref setScope, c.TypeArgs, out setScope)); + ResultAssert.InsufficientPermissions(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref setScope, "foo")); + RowCursor.Create(ref row, out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, "foo")); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor)); + ResultAssert.InsufficientPermissions(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref setScope, "foo")); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().DeleteSparse(ref row, ref setScope)); + + // Write a set of sets, successfully insert an empty set into it, and then try to write directly to the inner set. + Assert.IsTrue(this.layout.TryFind("prices", out c)); + RowCursor.Create(ref row, out setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref setScope, c.TypeArgs, out setScope)); + TypeArgument innerType = c.TypeArgs[0]; + TypeArgument itemType = innerType.TypeArgs[0]; + LayoutUniqueScope innerLayout = innerType.Type.TypeAs(); + RowCursor.Create(ref row, out RowCursor tempCursor1).Find(ref row, "prices.0"); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref tempCursor1, innerType.TypeArgs, out RowCursor innerScope)); + RowCursor.Create(ref row, out RowCursor tempCursor2).Find(ref row, "prices.0.0"); + ResultAssert.IsSuccess(itemType.Type.TypeAs().WriteSparse(ref row, ref tempCursor2, 1.0F)); + ResultAssert.IsSuccess(innerLayout.MoveField(ref row, ref innerScope, ref tempCursor2)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor1)); + Assert.IsTrue(setScope.MoveNext(ref row)); + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref setScope, out innerScope)); + ResultAssert.InsufficientPermissions(itemType.Type.TypeAs().WriteSparse(ref row, ref innerScope, 1.0F)); + ResultAssert.InsufficientPermissions(itemType.Type.TypeAs().DeleteSparse(ref row, ref innerScope)); + } + + [TestMethod] + [Owner("jthunter")] + public void PreventUniquenessViolations() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + Todo t1 = new Todo + { + Attendees = new List { "jason" }, + Projects = new List + { + Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), + }, + Prices = new List> + { + new List { 1.2F, 3.0F }, + }, + Shopping = new List + { + new ShoppingItem { Label = "milk", Count = 1 }, + }, + Work = new List> + { + Tuple.Create(false, 10000UL), + }, + }; + + this.WriteTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + + // Attempt to insert duplicate items in existing sets. + RowCursor root = RowCursor.Create(ref row); + Assert.IsTrue(this.layout.TryFind("attendees", out LayoutColumn c)); + root.Clone(out RowCursor setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, t1.Attendees[0])); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.Insert)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, t1.Attendees[0])); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref setScope, ref tempCursor, out RowCursor _)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.NotFound(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tempCursor, out string _)); + + Assert.IsTrue(this.layout.TryFind("projects", out c)); + root.Clone(out setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, t1.Projects[0])); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.Insert)); + + // Attempt to move a duplicate set into a set of sets. + Assert.IsTrue(this.layout.TryFind("prices", out c)); + root.Clone(out setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + TypeArgument innerType = c.TypeArgs[0]; + LayoutUniqueScope innerLayout = innerType.Type.TypeAs(); + root.Clone(out RowCursor tempCursor1).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref tempCursor1, innerType.TypeArgs, out RowCursor innerScope)); + foreach (float innerItem in t1.Prices[0]) + { + LayoutFloat32 itemLayout = innerType.TypeArgs[0].Type.TypeAs(); + root.Clone(out RowCursor tempCursor2).Find(ref row, "prices.0.0"); + ResultAssert.IsSuccess(itemLayout.WriteSparse(ref row, ref tempCursor2, innerItem)); + ResultAssert.IsSuccess(innerLayout.MoveField(ref row, ref innerScope, ref tempCursor2)); + } + + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor1, UpdateOptions.Insert)); + + // Attempt to move a duplicate UDT into a set of UDT. + Assert.IsTrue(this.layout.TryFind("shopping", out c)); + root.Clone(out setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + LayoutUDT udtLayout = c.TypeArgs[0].Type.TypeAs(); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(udtLayout.WriteScope(ref row, ref tempCursor, c.TypeArgs[0].TypeArgs, out RowCursor udtScope)); + TypedSetUnitTests.WriteShoppingItem(ref row, ref udtScope, c.TypeArgs[0].TypeArgs, t1.Shopping[0]); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.Insert)); + + // Attempt to move a duplicate tuple into a set of tuple. + Assert.IsTrue(this.layout.TryFind("work", out c)); + root.Clone(out setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + innerType = c.TypeArgs[0]; + LayoutIndexedScope tupleLayout = innerType.Type.TypeAs(); + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(tupleLayout.WriteScope(ref row, ref tempCursor, innerType.TypeArgs, out RowCursor tupleScope)); + ResultAssert.IsSuccess(innerType.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tupleScope, t1.Work[0].Item1)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(innerType.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref tupleScope, t1.Work[0].Item2)); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.Insert)); + } + + [TestMethod] + [Owner("jthunter")] + public void FindInSet() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + Todo t1 = new Todo + { + Attendees = new List { "jason", "janice", "joshua" }, + Prices = new List> + { + new List { 1.2F, 3.0F }, + new List { 4.1F, 5.7F }, + new List { 7.3F, 8.12F, 9.14F }, + }, + }; + + this.WriteTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + + // Attempt to find each item in turn. + RowCursor root = RowCursor.Create(ref row); + Assert.IsTrue(this.layout.TryFind("attendees", out LayoutColumn c)); + root.Clone(out RowCursor setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + for (int i = 0; i < t1.Attendees.Count; i++) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, t1.Attendees[i])); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref setScope, ref tempCursor, out RowCursor findScope)); + Assert.AreEqual(i, findScope.Index, $"Failed to find t1.Attendees[{i}]"); + } + + Assert.IsTrue(this.layout.TryFind("prices", out c)); + root.Clone(out setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + TypeArgument innerType = c.TypeArgs[0]; + TypeArgument itemType = innerType.TypeArgs[0]; + LayoutUniqueScope innerLayout = innerType.Type.TypeAs(); + for (int i = 0; i < t1.Prices.Count; i++) + { + root.Clone(out RowCursor tempCursor1).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref tempCursor1, innerType.TypeArgs, out RowCursor innerScope)); + for (int j = 0; j < t1.Prices[i].Count; j++) + { + root.Clone(out RowCursor tempCursor2).Find(ref row, "prices.0.0"); + ResultAssert.IsSuccess(itemType.Type.TypeAs().WriteSparse(ref row, ref tempCursor2, t1.Prices[i][j])); + ResultAssert.IsSuccess(innerLayout.MoveField(ref row, ref innerScope, ref tempCursor2)); + } + + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref setScope, ref tempCursor1, out RowCursor findScope)); + Assert.AreEqual(i, findScope.Index, $"Failed to find t1.Prices[{i}]"); + } + } + + [TestMethod] + [Owner("jthunter")] + public void UpdateInSet() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + List expected = new List + { + Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), + Guid.Parse("{7499C40E-7077-45C1-AE5F-3E384966B3B9}"), + Guid.Parse("{B7BC39C2-1A2D-4EAF-8F33-ED976872D876}"), + Guid.Parse("{DEA71ABE-3041-4CAF-BBD9-1A46D10832A0}"), + }; + + foreach (IEnumerable permutation in expected.Permute()) + { + Todo t1 = new Todo + { + Projects = new List(permutation), + }; + + this.WriteTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + + // Attempt to find each item in turn and then delete it. + RowCursor root = RowCursor.Create(ref row); + Assert.IsTrue(this.layout.TryFind("projects", out LayoutColumn c)); + root.Clone(out RowCursor setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + foreach (Guid elm in t1.Projects) + { + // Verify it is already there. + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, elm)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref setScope, ref tempCursor, value: out RowCursor _)); + + // Insert it again with update. + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, elm)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.Update)); + + // Insert it again with upsert. + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, elm)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor)); + + // Insert it again with insert (fail: exists). + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, elm)); + ResultAssert.Exists(c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.Insert)); + + // Insert it again with insert at (fail: disallowed). + root.Clone(out tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, elm)); + ResultAssert.TypeConstraint( + c.TypeAs().MoveField(ref row, ref setScope, ref tempCursor, UpdateOptions.InsertAt)); + } + } + } + + [TestMethod] + [Owner("jthunter")] + public void FindAndDelete() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + + List expected = new List + { + Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), + Guid.Parse("{7499C40E-7077-45C1-AE5F-3E384966B3B9}"), + Guid.Parse("{B7BC39C2-1A2D-4EAF-8F33-ED976872D876}"), + Guid.Parse("{DEA71ABE-3041-4CAF-BBD9-1A46D10832A0}"), + }; + + foreach (IEnumerable permutation in expected.Permute()) + { + Todo t1 = new Todo + { + Projects = new List(permutation), + }; + + this.WriteTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _), t1); + + // Attempt to update each item in turn and then update it with itself. + RowCursor root = RowCursor.Create(ref row); + Assert.IsTrue(this.layout.TryFind("projects", out LayoutColumn c)); + root.Clone(out RowCursor setScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref setScope, out setScope)); + foreach (Guid p in t1.Projects) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, p)); + ResultAssert.IsSuccess(c.TypeAs().Find(ref row, ref setScope, ref tempCursor, out RowCursor findScope)); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().DeleteSparse(ref row, ref findScope)); + } + } + + } + + + [TestMethod] + [Owner("jthunter")] + public void RowWriterTest() + { + RowBuffer row = new RowBuffer(TypedSetUnitTests.InitialRowSize); + + List expected = new List + { + Guid.Parse("{4674962B-CE11-4916-81C5-0421EE36F168}"), + Guid.Parse("{7499C40E-7077-45C1-AE5F-3E384966B3B9}"), + Guid.Parse("{B7BC39C2-1A2D-4EAF-8F33-ED976872D876}"), + Guid.Parse("{DEA71ABE-3041-4CAF-BBD9-1A46D10832A0}"), + }; + + foreach (IEnumerable permutation in expected.Permute()) + { + Todo t1 = new Todo + { + Projects = new List(permutation), + }; + + row.InitLayout(HybridRowVersion.V1, this.layout, this.resolver); + ResultAssert.IsSuccess(RowWriter.WriteBuffer(ref row, t1, TypedSetUnitTests.SerializeTodo)); + + // Update the existing Set by updating each item with itself. This ensures that the RowWriter has + // maintained the unique index correctly. + Assert.IsTrue(this.layout.TryFind("projects", out LayoutColumn c)); + RowCursor.Create(ref row, out RowCursor root); + root.Clone(out RowCursor projScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().ReadScope(ref row, ref projScope, out projScope)); + foreach (Guid item in t1.Projects) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, item)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref projScope, ref tempCursor)); + } + + Todo t2 = this.ReadTodo(ref row, ref RowCursor.Create(ref row, out RowCursor _)); + Assert.AreEqual(t1, t2); + } + } + + private static Result SerializeTodo(ref RowWriter writer, TypeArgument typeArg, Todo value) + { + if (value.Projects != null) + { + Assert.IsTrue(writer.Layout.TryFind("projects", out LayoutColumn c)); + Result r = writer.WriteScope( + "projects", + c.TypeArg, + value.Projects, + (ref RowWriter writer2, TypeArgument typeArg2, List value2) => + { + foreach (Guid item in value2) + { + ResultAssert.IsSuccess(writer2.WriteGuid(null, item)); + } + + return Result.Success; + }); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + private void WriteTodo(ref RowBuffer row, ref RowCursor root, Todo value) + { + LayoutColumn c; + + if (value.Attendees != null) + { + Assert.IsTrue(this.layout.TryFind("attendees", out c)); + root.Clone(out RowCursor attendScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref attendScope, c.TypeArgs, out attendScope)); + foreach (string item in value.Attendees) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, item)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref attendScope, ref tempCursor)); + } + } + + if (value.Projects != null) + { + Assert.IsTrue(this.layout.TryFind("projects", out c)); + root.Clone(out RowCursor projScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref projScope, c.TypeArgs, out projScope)); + foreach (Guid item in value.Projects) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, item)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref projScope, ref tempCursor)); + } + } + + if (value.Checkboxes != null) + { + Assert.IsTrue(this.layout.TryFind("checkboxes", out c)); + root.Clone(out RowCursor checkboxScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref checkboxScope, c.TypeArgs, out checkboxScope)); + foreach (bool item in value.Checkboxes) + { + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tempCursor, item)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref checkboxScope, ref tempCursor)); + } + } + + if (value.Prices != null) + { + Assert.IsTrue(this.layout.TryFind("prices", out c)); + root.Clone(out RowCursor pricesScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref pricesScope, c.TypeArgs, out pricesScope)); + foreach (List item in value.Prices) + { + Assert.IsTrue(item != null); + TypeArgument innerType = c.TypeArgs[0]; + LayoutUniqueScope innerLayout = innerType.Type.TypeAs(); + root.Clone(out RowCursor tempCursor1).Find(ref row, "prices.0"); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref tempCursor1, innerType.TypeArgs, out RowCursor innerScope)); + foreach (float innerItem in item) + { + LayoutFloat32 itemLayout = innerType.TypeArgs[0].Type.TypeAs(); + root.Clone(out RowCursor tempCursor2).Find(ref row, "prices.0.0"); + ResultAssert.IsSuccess(itemLayout.WriteSparse(ref row, ref tempCursor2, innerItem)); + ResultAssert.IsSuccess(innerLayout.MoveField(ref row, ref innerScope, ref tempCursor2)); + } + + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref pricesScope, ref tempCursor1)); + } + } + + if (value.Nested != null) + { + Assert.IsTrue(this.layout.TryFind("nested", out c)); + root.Clone(out RowCursor nestedScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref nestedScope, c.TypeArgs, out nestedScope)); + foreach (List> item in value.Nested) + { + Assert.IsTrue(item != null); + TypeArgument in2Type = c.TypeArgs[0]; + LayoutUniqueScope in2Layout = in2Type.Type.TypeAs(); + root.Clone(out RowCursor tempCursor1).Find(ref row, "prices.0"); + ResultAssert.IsSuccess(in2Layout.WriteScope(ref row, ref tempCursor1, in2Type.TypeArgs, out RowCursor in2Scope)); + foreach (List item2 in item) + { + Assert.IsTrue(item2 != null); + TypeArgument in3Type = in2Type.TypeArgs[0]; + LayoutUniqueScope in3Layout = in3Type.Type.TypeAs(); + root.Clone(out RowCursor tempCursor2).Find(ref row, "prices.0.0"); + ResultAssert.IsSuccess(in3Layout.WriteScope(ref row, ref tempCursor2, in3Type.TypeArgs, out RowCursor in3Scope)); + foreach (int innerItem in item2) + { + LayoutInt32 itemLayout = in3Type.TypeArgs[0].Type.TypeAs(); + root.Clone(out RowCursor tempCursor3).Find(ref row, "prices.0.0.0"); + ResultAssert.IsSuccess(itemLayout.WriteSparse(ref row, ref tempCursor3, innerItem)); + ResultAssert.IsSuccess(in3Layout.MoveField(ref row, ref in3Scope, ref tempCursor3)); + } + + ResultAssert.IsSuccess(in2Layout.MoveField(ref row, ref in2Scope, ref tempCursor2)); + } + + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref nestedScope, ref tempCursor1)); + } + } + + if (value.Shopping != null) + { + Assert.IsTrue(this.layout.TryFind("shopping", out c)); + root.Clone(out RowCursor shoppingScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref shoppingScope, c.TypeArgs, out shoppingScope)); + foreach (ShoppingItem item in value.Shopping) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutUDT innerLayout = innerType.Type.TypeAs(); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref tempCursor, innerType.TypeArgs, out RowCursor itemScope)); + TypedSetUnitTests.WriteShoppingItem(ref row, ref itemScope, innerType.TypeArgs, item); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref shoppingScope, ref tempCursor)); + } + } + + if (value.Work != null) + { + Assert.IsTrue(this.layout.TryFind("work", out c)); + root.Clone(out RowCursor workScope).Find(ref row, c.Path); + ResultAssert.IsSuccess(c.TypeAs().WriteScope(ref row, ref workScope, c.TypeArgs, out workScope)); + foreach (Tuple item in value.Work) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutIndexedScope innerLayout = innerType.Type.TypeAs(); + root.Clone(out RowCursor tempCursor).Find(ref row, Utf8String.Empty); + ResultAssert.IsSuccess(innerLayout.WriteScope(ref row, ref tempCursor, innerType.TypeArgs, out RowCursor tupleScope)); + ResultAssert.IsSuccess(innerType.TypeArgs[0].Type.TypeAs().WriteSparse(ref row, ref tupleScope, item.Item1)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(innerType.TypeArgs[1].Type.TypeAs().WriteSparse(ref row, ref tupleScope, item.Item2)); + ResultAssert.IsSuccess(c.TypeAs().MoveField(ref row, ref workScope, ref tempCursor)); + } + } + } + + private Todo ReadTodo(ref RowBuffer row, ref RowCursor root) + { + Todo value = new Todo(); + + Assert.IsTrue(this.layout.TryFind("attendees", out LayoutColumn c)); + root.Clone(out RowCursor tagsScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref tagsScope, out tagsScope) == Result.Success) + { + value.Attendees = new List(); + while (tagsScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tagsScope, out string item)); + value.Attendees.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("projects", out c)); + root.Clone(out RowCursor projScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref projScope, out projScope) == Result.Success) + { + value.Projects = new List(); + while (projScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref projScope, out Guid item)); + value.Projects.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("checkboxes", out c)); + root.Clone(out RowCursor checkboxScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref checkboxScope, out checkboxScope) == Result.Success) + { + value.Checkboxes = new List(); + while (checkboxScope.MoveNext(ref row)) + { + ResultAssert.IsSuccess(c.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref checkboxScope, out bool item)); + value.Checkboxes.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("prices", out c)); + root.Clone(out RowCursor pricesScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref pricesScope, out pricesScope) == Result.Success) + { + value.Prices = new List>(); + TypeArgument innerType = c.TypeArgs[0]; + LayoutUniqueScope innerLayout = innerType.Type.TypeAs(); + while (pricesScope.MoveNext(ref row)) + { + List item = new List(); + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref pricesScope, out RowCursor innerScope)); + while (innerScope.MoveNext(ref row)) + { + LayoutFloat32 itemLayout = innerType.TypeArgs[0].Type.TypeAs(); + ResultAssert.IsSuccess(itemLayout.ReadSparse(ref row, ref innerScope, out float innerItem)); + item.Add(innerItem); + } + + value.Prices.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("nested", out c)); + root.Clone(out RowCursor nestedScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref nestedScope, out nestedScope) == Result.Success) + { + value.Nested = new List>>(); + TypeArgument in2Type = c.TypeArgs[0]; + LayoutUniqueScope in2Layout = in2Type.Type.TypeAs(); + while (nestedScope.MoveNext(ref row)) + { + List> item = new List>(); + ResultAssert.IsSuccess(in2Layout.ReadScope(ref row, ref nestedScope, out RowCursor in2Scope)); + while (in2Scope.MoveNext(ref row)) + { + TypeArgument in3Type = in2Type.TypeArgs[0]; + LayoutUniqueScope in3Layout = in3Type.Type.TypeAs(); + List item2 = new List(); + ResultAssert.IsSuccess(in3Layout.ReadScope(ref row, ref in2Scope, out RowCursor in3Scope)); + while (in3Scope.MoveNext(ref row)) + { + LayoutInt32 itemLayout = in3Type.TypeArgs[0].Type.TypeAs(); + ResultAssert.IsSuccess(itemLayout.ReadSparse(ref row, ref in3Scope, out int innerItem)); + item2.Add(innerItem); + } + + item.Add(item2); + } + + value.Nested.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("shopping", out c)); + root.Clone(out RowCursor shoppingScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref shoppingScope, out shoppingScope) == Result.Success) + { + value.Shopping = new List(); + while (shoppingScope.MoveNext(ref row)) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutUDT innerLayout = innerType.Type.TypeAs(); + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref shoppingScope, out RowCursor matchScope)); + ShoppingItem item = TypedSetUnitTests.ReadShoppingItem(ref row, ref matchScope); + value.Shopping.Add(item); + } + } + + Assert.IsTrue(this.layout.TryFind("work", out c)); + root.Clone(out RowCursor workScope).Find(ref row, c.Path); + if (c.TypeAs().ReadScope(ref row, ref workScope, out workScope) == Result.Success) + { + value.Work = new List>(); + while (workScope.MoveNext(ref row)) + { + TypeArgument innerType = c.TypeArgs[0]; + LayoutIndexedScope innerLayout = innerType.Type.TypeAs(); + + ResultAssert.IsSuccess(innerLayout.ReadScope(ref row, ref workScope, out RowCursor tupleScope)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(innerType.TypeArgs[0].Type.TypeAs().ReadSparse(ref row, ref tupleScope, out bool item1)); + Assert.IsTrue(tupleScope.MoveNext(ref row)); + ResultAssert.IsSuccess(innerType.TypeArgs[1].Type.TypeAs().ReadSparse(ref row, ref tupleScope, out ulong item2)); + value.Work.Add(Tuple.Create(item1, item2)); + } + } + + return value; + } + + private static void WriteShoppingItem(ref RowBuffer row, ref RowCursor matchScope, TypeArgumentList typeArgs, ShoppingItem m) + { + Layout matchLayout = row.Resolver.Resolve(typeArgs.SchemaId); + Assert.IsTrue(matchLayout.TryFind("label", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().WriteVariable(ref row, ref matchScope, c, m.Label)); + Assert.IsTrue(matchLayout.TryFind("count", out c)); + ResultAssert.IsSuccess(c.TypeAs().WriteFixed(ref row, ref matchScope, c, m.Count)); + } + + private static ShoppingItem ReadShoppingItem(ref RowBuffer row, ref RowCursor matchScope) + { + Layout matchLayout = matchScope.Layout; + ShoppingItem m = new ShoppingItem(); + Assert.IsTrue(matchLayout.TryFind("label", out LayoutColumn c)); + ResultAssert.IsSuccess(c.TypeAs().ReadVariable(ref row, ref matchScope, c, out m.Label)); + Assert.IsTrue(matchLayout.TryFind("count", out c)); + ResultAssert.IsSuccess(c.TypeAs().ReadFixed(ref row, ref matchScope, c, out m.Count)); + return m; + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class Todo + { + public List Attendees; + public List Projects; + public List Checkboxes; + public List> Prices; + public List>> Nested; + public List Shopping; + public List> Work; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is Todo todo && this.Equals(todo); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = 0; + hashCode = (hashCode * 397) ^ (this.Attendees?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Projects?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Checkboxes?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Prices?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Nested?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Shopping?.GetHashCode() ?? 0); + hashCode = (hashCode * 397) ^ (this.Work?.GetHashCode() ?? 0); + return hashCode; + } + } + + private static bool NestedNestedSetEquals(List>> left, List>> right) + { + if (left.Count != right.Count) + { + return false; + } + + for (int i = 0; i < left.Count; i++) + { + if (!Todo.NestedSetEquals(left[i], right[i])) + { + return false; + } + } + + return true; + } + + private static bool NestedSetEquals(List> left, List> right) + { + if (left.Count != right.Count) + { + return false; + } + + for (int i = 0; i < left.Count; i++) + { + if (!Todo.SetEquals(left[i], right[i])) + { + return false; + } + } + + return true; + } + + private static bool SetEquals(List left, List right) + { + if (left.Count != right.Count) + { + return false; + } + + foreach (T item in left) + { + if (!right.Contains(item)) + { + return false; + } + } + + return true; + } + + private bool Equals(Todo other) + { + return (object.ReferenceEquals(this.Attendees, other.Attendees) || + ((this.Attendees != null) && (other.Attendees != null) && Todo.SetEquals(this.Attendees, other.Attendees))) && + (object.ReferenceEquals(this.Projects, other.Projects) || + ((this.Projects != null) && (other.Projects != null) && Todo.SetEquals(this.Projects, other.Projects))) && + (object.ReferenceEquals(this.Checkboxes, other.Checkboxes) || + ((this.Checkboxes != null) && (other.Checkboxes != null) && Todo.SetEquals(this.Checkboxes, other.Checkboxes))) && + (object.ReferenceEquals(this.Prices, other.Prices) || + ((this.Prices != null) && (other.Prices != null) && Todo.NestedSetEquals(this.Prices, other.Prices))) && + (object.ReferenceEquals(this.Nested, other.Nested) || + ((this.Nested != null) && (other.Nested != null) && Todo.NestedNestedSetEquals(this.Nested, other.Nested))) && + (object.ReferenceEquals(this.Shopping, other.Shopping) || + ((this.Shopping != null) && (other.Shopping != null) && Todo.SetEquals(this.Shopping, other.Shopping))) && + (object.ReferenceEquals(this.Work, other.Work) || + ((this.Work != null) && (other.Work != null) && Todo.SetEquals(this.Work, other.Work))); + } + } + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + private sealed class ShoppingItem + { + public string Label; + public byte Count; + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (object.ReferenceEquals(this, obj)) + { + return true; + } + + return obj is ShoppingItem shoppingItem && this.Equals(shoppingItem); + } + + public override int GetHashCode() + { + unchecked + { + return (this.Label.GetHashCode() * 397) ^ this.Count.GetHashCode(); + } + } + + private bool Equals(ShoppingItem other) + { + return this.Label == other.Label && this.Count == other.Count; + } + } + } +} diff --git a/dotnet/src/HybridRow.Tests.Unit/UpdateOptionsUnitTests.cs b/dotnet/src/HybridRow.Tests.Unit/UpdateOptionsUnitTests.cs new file mode 100644 index 0000000..b5e4135 --- /dev/null +++ b/dotnet/src/HybridRow.Tests.Unit/UpdateOptionsUnitTests.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class UpdateOptionsUnitTests + { + [TestMethod] + [Owner("jthunter")] + public void UpdateOptionsTest() + { + Assert.AreEqual((int)RowOptions.None, (int)UpdateOptions.None); + Assert.AreEqual((int)RowOptions.Update, (int)UpdateOptions.Update); + Assert.AreEqual((int)RowOptions.Insert, (int)UpdateOptions.Insert); + Assert.AreEqual((int)RowOptions.Upsert, (int)UpdateOptions.Upsert); + Assert.AreEqual((int)RowOptions.InsertAt, (int)UpdateOptions.InsertAt); + } + } +} diff --git a/dotnet/src/HybridRow/DefaultSpanResizer.cs b/dotnet/src/HybridRow/DefaultSpanResizer.cs new file mode 100644 index 0000000..453f8d6 --- /dev/null +++ b/dotnet/src/HybridRow/DefaultSpanResizer.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + using System.Diagnostics.CodeAnalysis; + + public class DefaultSpanResizer : ISpanResizer + { + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly DefaultSpanResizer Default = new DefaultSpanResizer(); + + private DefaultSpanResizer() + { + } + + /// + public Span Resize(int minimumLength, Span buffer = default) + { + Span next = new Memory(new T[Math.Max(minimumLength, buffer.Length)]).Span; + if (!buffer.IsEmpty && next.Slice(0, buffer.Length) != buffer) + { + buffer.CopyTo(next); + } + + return next; + } + } +} diff --git a/dotnet/src/HybridRow/Docs/Glossary.md b/dotnet/src/HybridRow/Docs/Glossary.md new file mode 100644 index 0000000..dc870d3 --- /dev/null +++ b/dotnet/src/HybridRow/Docs/Glossary.md @@ -0,0 +1,71 @@ +This *glossary* defines some terms used either in the HybirdRow Library code, its +documentation, or in its applications. + +The definitions here are meant only to provide a common understanding when +implementing or consuming the HybridRow Library. + +[[_TOC_]] + +# Glossary + +## General Terms + +###### DDL +A Data Definition Language operation is one that defines a new [schema](#schema) or +redefines (ALTERs) an existing [schema](#schema). + +Since HybridRow [schema](#schema) are themselves immutable, a +DDL operation that ALTERs a [schema](#schema) always defines a new [schema](#schema) + with a distinct [SchemaId](#schemaid). By convention the new [schema](#schema) has +the same name as previous schema being ALTERed and a new SchemaId whose absolutely value +is monotonically increasing relative to the old schema. See [SchemaId.md](./SchemaId.md) +for details on how [SchemaId](#schemaid) are allocated. + +###### Namespace +A set of [schema](#schema) with non-overlapping [SchemaId](#schemaid). + +###### Schema +Describes the logical structure of a row at a particular point in time (relative to the +DDL operation history). + +###### SchemaHash +A 128-bit hash of a HybridRow [schema](#schema). The hash captures only the logical +elements of a schema. Whitespace elements such as formatting or comments have no impact +on the hash. See [SchemaHash.md](./SchemaHash.md) for more details. + +###### SchemaId +An integer that uniquely defines a particular version of a schema within a +Namespace. + + +## Schema Versioning Terms + +###### Latest (Schema) Version +The [SchemaId](#schemaid) of the latest known version of the schema (from the Backend's perspective). + +###### Row (Schema) Version +The [SchemaId](#schemaid) at which the stored row was encoded. When a row is read it is +**upgraded** through a process called **row upgrade** when the +`Row Version < Latest Version`. + +###### Target (Schema) Version +An operation performed by the Front End (FE) or Client (referred to collectively as FE +below) is done in the context of its understanding of the current schema. + +The FE's view may trail the true Latest Version (`Target Version < Latest Version`) +if the FE's schema cache is stale. The FE's view may lead the true Latest Version +(`Target Version > Latest Version`) if a DDL operation has happened but has not +yet propagated to the specific BE in question (propagation is asynchronous and +concurrent). + +The FE provides its view as the Target Version for each request. A Target Version +is always relative to a request and describes the version targeted by that request. +When the `Target Version < Latest Version`, the BE can still accept the request by +**upgrading** the request *before* applying it to the row. When the +`Target Version > Latest Version`, the BE must reject the request. +The FE is free to resubmit the request some time later in hopes that the BE has by then +seen the DDL operation and subsequently raised the Latest Version to match. The BE will, +of course, continue to reject requests whose Target Version is greater than its +Latest Version until the relavant DDL operation has successfully propagated. +This protection prevents the BE from writing rows that it would be unable to +immediately read. diff --git a/dotnet/src/HybridRow/Docs/Grammar.md b/dotnet/src/HybridRow/Docs/Grammar.md new file mode 100644 index 0000000..daaa0f0 --- /dev/null +++ b/dotnet/src/HybridRow/Docs/Grammar.md @@ -0,0 +1,109 @@ +This document contains a brief EBNF-like grammar for describing the structure of a Hybrid Row. + +[[_TOC_]] + +# HybridRow Format +HybridRows are described by the following +[EBNF grammar](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form): + +````ebnf +hybrid_row = hybrid_header, hybrid_body; +hybrid_header = version, schema_id; +hybrid_body = [schematized_fragment], { sparse_fragment }; +schematized_fragment = presence_and_bool_bits, fixed_fields, var_fields; +version = uint8; +schema_id = int32; +```` + + +The presence of a *Schematized Fragment* is defined by the schema referenced by the schema_id. +If the schema defines a Schematized Fragment, then one MUST be present. If it does NOT define +a Schematized Fragment, then one MUST NOT be present. The order of fields within a +Schematized Fragment, both fixed and variable length fields, is strictly defined by the schema +and all fields MUST be present, or its absence MUST be indicated by unsetting the corresponding +presence bit in the presence_bits field of the fragment. A presence bit is defined for all +nullable schematized fields. Non-nullable fields are **always** present, and if unwritten contain +their type's default value. + +The formal specification of the Schematized Fragment is: + +```ebnf +presence_and_bool_bits = ? 0+ bits, one for each nullable schematized field. Additionally one for each boolean field ?; +fixed_fields = { fixed_field }; +fixed_field = finite_field | bounded_field; +finite_field = literal_null_field + | bool_field + | integer_field + | float_field + | guid_field + | datetime_field + | object_id_field; +bounded_field = string_field | binary_field; +var_fields = { var_field }; +var_field = field_length, bounded_field | varint_field; +```` + +With the following primitive type definitions. Note, unless otherwise states, all values are stored little-endian. + +````ebnf +literal_null_field = ? no encoded payload ?; +bool_field = ? 1 byte, 0 FALSE, 1 TRUE ?; +integer_field = int8 | int16 | int32 | int64 | uint8 | uint16 | uint32 | uint64; +float_field = + float32 (? 4 byte IEEE 754 floating point value ?) + | float64 (? 8 byte IEEE 754 floating point value ?) + | float128 (? 16 byte IEEE 754 floating point value ?) + | decimal; (? 16 byte System.Decimal value ?) +guid_field = ? 16 byte, little-endian ?; +datetime_field = precise_datatime_field | unix_datetime_field; +precise_datatime_field = ? 8 byte, 100ns since 00:00:00, January 1, 1 CE UTC ?; +unix_datetime_field = ? 8 byte, milliseconds since Unix Epoch (midnight, January 1, 1970 UTC) ?; +object_id_field = ? 12 unsigned bytes (in big-endian order)?; +string_field = ? UTF-8, not null-terminated ?; +binary_field = ? unsigned bytes ?; +field_length = varuint; +varint_field = varint | varuint; +varint = ? varint – variable length signed integer encoding, sign-bit rotated ?; +varuint = ? varuint – variable length unsigned integer encoding ?; +```` + +Additionally, a row may contain zero or more *Sparse Fragments*. The structure of Sparse Fragment +may be described in full or in part within the schema. Describing Sparse Fragments within the schema +allows both for optional schema validation and path interning (See Paths). + +And Sparse Fragments: + +```ebnf +sparse_fragment = { sparse_field }; +sparse_field = type, path, sparse_value; +type = type_code | generic_type; +generic_type = + typed_array_type_code, type + | nullable_type_code, type + | typed_tuple_type_code, type, {type} + | typed_set_type_code, type + | typed_map_type_code, type, type + | hybrid_row_type_code, schema_id; +sparse_value = + null_field + | finite_field + | var_field + | array_field (? typed arrays Generic N = 1 ?) + | obj_field + | tuple_field (? typed tuples are Generic N ?) + | set_field (? Generic N = 1 ?) + | map_field (? Generic N = 2 ?) + | hybrid_body; (? Generic N = 1 ?) +null_field = (? empty production ?); +array_field = typed_array_field | sparse_array_field; +typed_array_field = field_count, { sparse_value }; +sparse_array_field = { type, sparse_value }, scope_end_symbol; +obj_field = { sparse_field }, scope_end_symbol; +nullable_field = bool_field, sparse_value; +tuple_field = sparse_value, {sparse_value}; +set_field = field_count, { sparse_value }; +map_field = field_count, { map_field_body }; +map_field_body = sparse_value (? key ?), sparse_value (? value ?); +path = ? dictionary (@) encoded varint ? | field_length, string_lob_field; + +```` diff --git a/dotnet/src/HybridRow/Docs/RecordIO.md b/dotnet/src/HybridRow/Docs/RecordIO.md new file mode 100644 index 0000000..751686a --- /dev/null +++ b/dotnet/src/HybridRow/Docs/RecordIO.md @@ -0,0 +1,38 @@ +RecordIO refers to a class of streaming file formats that consist of +linear sequences of flat records. The metadata describing the records and the +encoding format of the records vary between different RecordIO incarnations. +This document describes a HybridRow RecordIO format. + +[[_TOC_]] + +# HybridRow RecordIO Stream Format +HybridRow RecordIO streams are described by the following +[EBNF grammar](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form): + +```ebnf +record_io_stream = record_io_segment, {record_io_segment}; +record_io_segment = record_io_header, [record]; +record_io_header = "RecordIO Segment HybridRow"; +record = record_header, record_body; +record_header = "RecordIO Record HybridRow"; +record_body = "Any HybridRow enocded row"; +``` + +A HybridRow RecordIO stream consists of one or more RecordIO segments, each +segment consisting of a RecordIO segment header followed by zero or more +records. + +Each record consists of a record header which includes both the length +(in bytes) of the record body and an optional CRC. The record body can be any +encoded HybridRow. + +Record bodies (serialized HybridRows) are limited to 2GB in length. + +If a schema namespace is provided in the segment header, then all records +within the segment **MUST** conform to a schema type defined within that +schema namespace. + + +# RecordIO Schema +The segment and record headers are themselves HybridRow serialized values +described by [RecordIO System Schema](../SystemSchemas/RecordIOSchema.json). diff --git a/dotnet/src/HybridRow/Docs/SchemaHash.md b/dotnet/src/HybridRow/Docs/SchemaHash.md new file mode 100644 index 0000000..46faa33 --- /dev/null +++ b/dotnet/src/HybridRow/Docs/SchemaHash.md @@ -0,0 +1,53 @@ +`SchemaHash` is a 128-bit hash of a HybridRow schema. + +[[_TOC_]] + +# Schema Evolution +During schema evolution scenarios (e.g. DDL) it is necessary to ensure that the +historical record of previous schema versions has not been improperly altered +and that the new schema introduced (e.g. as a result of the DDL operation) are +a proper and valid superset of that history. + +If the historical record were altered or truncated then existing "old" rows +within the table might no longer be parsable, or their values may be +misinterpreted. Because of potential changes to the schema language version, +and non-determinism in the natural encoding of JSON-based schemas (e.g. +ordering, commas, comments) textual comparison is both insufficient and +incorrect. A logical comparison of the relevant schema structure using only +canonical formulizations that will be true across schema language versions +should be used. + +`SchemaHash` defines exactly and only the necessary structural elements and +thus implements a method for calculating the logical schema version (hash) of a +schema given a namespace that contains that schema and its dependent +closure of types. + +Notes: +* The provided Namespace may contain additional schemas not related to the + given schema (including other versions of that schema or other versions of + its dependent types). +* `SchemaHash` applies to a particular schema at a time, not an entire + namespace of schema. +* `SchemaHash` incorporates recursively the `SchemaHash` of each nested + schema (aka UDTs) that appear in the schema closure of the type. Thus a + `SchemaHash` provides a snapshot *version* that uniquely describes a row's + metadata at a specific point in time. + +# Algorithm +`SchemaHash` is computed as an accumulated Murmur hash. The Little Endian, +x64, 128-bit [MurmurHash3](https://en.wikipedia.org/wiki/MurmurHash) algorithm +is to be used. The hash is computed over the relevant structural elements. + +The hash is accumulated by passing the current accumulated +hash as the seed for the next round of hashing, thus chaining the hash results +until all structures are hashed. + +All structural elements of the schema are hashed as individual blocks encoding +each block as the little-endian byte sequence with the following caveats: + +* null values are skipped (contribute nothing to the accumulation). +* strings are hashed as their canonicalized UTF8 byte sequence without either + the length or null-termination. +* bools are hashed as a single byte: 1 for true and 0 for false. +* Lists are hashed by hashing their element in the ordered sequence in which they + appear in the SDL. Order matters. Empty lists are treated as nulls. \ No newline at end of file diff --git a/dotnet/src/HybridRow/Docs/SchemaId.md b/dotnet/src/HybridRow/Docs/SchemaId.md new file mode 100644 index 0000000..7bd6590 --- /dev/null +++ b/dotnet/src/HybridRow/Docs/SchemaId.md @@ -0,0 +1,49 @@ +`SchemaId` are 32-bit unique identifiers used in Hybrid Row to uniquely +reference a Schema type within a Schema Namespace. Schema have names, but +since multiple revisions of the same Schema may be defined within the same +Schema Namespace, the `SchemaId` distiguishes the revisions from each other. + +[[_TOC_]] + +# Hybrid Row Runtime +There is no `SchemaId` allocation policy imposed directly by the Hybrid Row +runtime, however, the runtime does require that all policies meet the +following requirements: + + * All `SchemaId` **MUST** be unique **within** a Schema Namespace. + * The `SchemaId` of `0` is reserved as the `Invalid` `SchemaId` and + must never be assigned. + + +# Cosmos DB +This section describes the set of convention used for `SchemaId` allocation +by [Azure Cosmos DB](https://azure.microsoft.com/en-us/services/cosmos-db/). + +Cosmos DB's `SchemaId` policy sub-divides the available 32-bit numeric address +space for `SchemaId` into distinct non-overlapping regions and assigns distinct +semantics for each region: + +Name | Range | Description +--- | --- | --- +Invalid | **0** | Reserved (by runtime) [\*](#bugs-and-known-issues) +Table Schema | [$-1000000$ .. $-1$] | Each revision of the table schema has a distinct *monotonically decreasing* numeric value. +UDT Schema | [$1$ .. $1000000$] | Each revision of each UDT schema has a distinct *monotonically increasing* numeric value. +System Schema | [$2147473648$ .. $2147483647$]

[`Int32.Max` - $9,999$ .. `Int32.Max`] | Reserved for system defined schema types. +Dynamic Schema | [$2146473647$ .. $2147473647$]

[`Int32.Max` - $1,010,000$ .. `Int32.Max` - $10,000$] | Reserved for context-specific schema generated dynamically (e.g. result set schema scoped to a channel.) +App-Specific Schema | [$2145473647$ .. $2146473647$]

[`Int32.Max` - $2,010,000$ .. `Int32.Max` - $1,009,999$] | Reserved for app-specific schema. (e.g. Batch API schema used by the Cosmos DB application/sdk.) + +## Monotonicity +Cosmos DB allows for the evolution of Schema over time. As Schema evolve new +revisions of their Schema are committed to the Schema Namespace. Because +existing encoded rows remain in the store referencing older revisions, all +revisions of a given Schema referenced by at least one existing row **MUST** +be retained in the Schema Namespace. As Schema are evolved, later revisions +of a Schema are assigned `SchemaId` whose (*absolute*) value is larger than +any previous revision of that Schema. The latest revision of any given Schema +is the Schema within the Schema Namespace with both a matching name and the +largest (*absolute*) `SchemaId` value. + + +## Bugs and Known Issues +A bug in the initial Cassandra GA code allocated some type schemas using the +`SchemaId` `0` accidentally. \ No newline at end of file diff --git a/dotnet/src/HybridRow/Docs/SystemSchema.md b/dotnet/src/HybridRow/Docs/SystemSchema.md new file mode 100644 index 0000000..35c1a63 --- /dev/null +++ b/dotnet/src/HybridRow/Docs/SystemSchema.md @@ -0,0 +1,27 @@ +System Schema are globally available HybridRow Schema definitions. This +document summarizes the available schema namespaces and their reserved SchemaId +allocated from the [System Schema](./SchemaId.md) address space. + +[[_TOC_]] + +# System Schema Catalog +The following are System Schema namespaces defined by the HybridRow runtime: + +* [**Microsoft.Azure.Cosmos.HybridRow.RecordIO**](./RecordIO.md): + Defines types used in streaming record-oriented files containing HybridRows. + +# SchemaId Reserved by the HybridRow Runtime + +* $2147473648$ - RecordIO Segment +* $2147473649$ - RecordIO Record +* $2147473650$ - Empty Schema + +# SchemaId Reserved by the Cosmos DB application +*These must be within [2145473647..2146473647]* + +* $2145473647$ - HybridRow Query Response +* $2145473648$ - Batch API Operation +* $2145473649$ - Batch API Result +* $2145473650$ - Patch Request +* $2145473651$ - Patch Operation +* $2145473652$ - Json Schema \ No newline at end of file diff --git a/dotnet/src/HybridRow/Float128.cs b/dotnet/src/HybridRow/Float128.cs new file mode 100644 index 0000000..30ae140 --- /dev/null +++ b/dotnet/src/HybridRow/Float128.cs @@ -0,0 +1,57 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1051 // Do not declare visible instance fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System.Diagnostics; + using System.Runtime.InteropServices; + + /// An IEEE 128-bit floating point value. + /// + /// A binary integer decimal representation of a 128-bit decimal value, supporting 34 decimal digits of + /// significand and an exponent range of -6143 to +6144. + /// + /// + /// Source Link + /// + /// Wikipedia: + /// https://en.wikipedia.org/wiki/Decimal128_floating-point_format + /// + /// The spec: https://ieeexplore.ieee.org/document/4610935 + /// + /// Decimal Encodings: http://speleotrove.com/decimal/decbits.html + /// + /// + /// + [DebuggerDisplay("{" + nameof(Float128.Low) + "," + nameof(Float128.High) + "}")] + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public readonly struct Float128 + { + /// The size (in bytes) of a . + public const int Size = sizeof(long) + sizeof(long); + + /// + /// The low-order 64 bits of the IEEE 754-2008 128-bit decimal floating point, using the BID + /// encoding scheme. + /// + public readonly long Low; + + /// + /// The high-order 64 bits of the IEEE 754-2008 128-bit decimal floating point, using the BID + /// encoding scheme. + /// + public readonly long High; + + /// Initializes a new instance of the struct. + /// the high-order 64 bits. + /// the low-order 64 bits. + public Float128(long high, long low) + { + this.High = high; + this.Low = low; + } + } +} diff --git a/dotnet/src/HybridRow/GlobalSuppressions.cs b/dotnet/src/HybridRow/GlobalSuppressions.cs new file mode 100644 index 0000000..a588984 --- /dev/null +++ b/dotnet/src/HybridRow/GlobalSuppressions.cs @@ -0,0 +1,9 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +[assembly: + System.Diagnostics.CodeAnalysis.SuppressMessage( + "StyleCop.CSharp.LayoutRules", + "SA1512:Single-line comments should not be followed by blank line", + Justification = "Refactoring")] diff --git a/dotnet/src/HybridRow/HybridRowHeader.cs b/dotnet/src/HybridRow/HybridRowHeader.cs new file mode 100644 index 0000000..8ed2587 --- /dev/null +++ b/dotnet/src/HybridRow/HybridRowHeader.cs @@ -0,0 +1,33 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System.Runtime.InteropServices; + + /// Describes the header the precedes all valid Hybrid Rows. + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public readonly struct HybridRowHeader + { + /// Size (in bytes) of a serialized header. + public const int Size = sizeof(HybridRowVersion) + SchemaId.Size; + + /// + /// Initializes a new instance of the struct. + /// + /// The version of the HybridRow library used to write this row. + /// The unique identifier of the schema whose layout was used to write this row. + public HybridRowHeader(HybridRowVersion version, SchemaId schemaId) + { + this.Version = version; + this.SchemaId = schemaId; + } + + /// The version of the HybridRow library used to write this row. + public HybridRowVersion Version { get; } + + /// The unique identifier of the schema whose layout was used to write this row. + public SchemaId SchemaId { get; } + } +} diff --git a/dotnet/src/HybridRow/HybridRowVersion.cs b/dotnet/src/HybridRow/HybridRowVersion.cs new file mode 100644 index 0000000..aa9f21f --- /dev/null +++ b/dotnet/src/HybridRow/HybridRowVersion.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1028 // Enum Storage should be Int32 + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + /// Versions of HybridRow. + /// A version from this list MUST be inserted in the version BOM at the beginning of all rows. + public enum HybridRowVersion : byte + { + Invalid = 0, + + /// Initial version of the HybridRow format. + V1 = 0x81, + } +} diff --git a/dotnet/src/HybridRow/IO/IRowSerializable.cs b/dotnet/src/HybridRow/IO/IRowSerializable.cs new file mode 100644 index 0000000..ba7a91f --- /dev/null +++ b/dotnet/src/HybridRow/IO/IRowSerializable.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.IO +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + /// + /// A type may implement this interface to support serialization into a HybridRow. + /// + public interface IRowSerializable + { + /// + /// Writes the current instance into the row. + /// + /// A writer for the current row scope. + /// The schematized layout type, if a schema is available. + /// Success if the write is successful, the error code otherwise. + Result Write(ref RowWriter writer, TypeArgument typeArg); + } +} diff --git a/dotnet/src/HybridRow/IO/RowReader.cs b/dotnet/src/HybridRow/IO/RowReader.cs new file mode 100644 index 0000000..3276cc8 --- /dev/null +++ b/dotnet/src/HybridRow/IO/RowReader.cs @@ -0,0 +1,1027 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1034 // Nested types should not be visible + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.IO +{ + using System; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + /// A forward-only, streaming, field reader for . + /// + /// A allows the traversal in a streaming, left to right fashion, of + /// an entire HybridRow. The row's layout provides decoding for any schematized portion of the row. + /// However, unschematized sparse fields are read directly from the sparse segment with or without + /// schematization allowing all fields within the row, both known and unknown, to be read. + /// + /// Modifying a invalidates any reader or child reader associated with it. In + /// general 's should not be mutated while being enumerated. + /// + public ref struct RowReader + { + private readonly int schematizedCount; + private readonly ReadOnlySpan columns; + private RowBuffer row; + + // State that can be checkpointed. + private States state; + private int columnIndex; + private RowCursor cursor; + + /// Initializes a new instance of the struct. + /// The row to be read. + /// The scope whose fields should be enumerated. + /// + /// A instance traverses all of the top-level fields of a given + /// scope. If the root scope is provided then all top-level fields in the row are enumerated. Nested + /// child instances can be access through the method + /// to process nested content. + /// + private RowReader(ref RowBuffer row, in RowCursor scope) + { + this.cursor = scope; + this.row = row; + this.columns = this.cursor.layout.Columns; + this.schematizedCount = this.cursor.layout.NumFixed + this.cursor.layout.NumVariable; + + this.state = States.None; + this.columnIndex = -1; + } + + /// Initializes a new instance of the struct. + /// The row to be read. + /// The scope whose fields should be enumerated. + /// + /// A instance traverses all of the top-level fields of a given + /// scope. If the root scope is provided then all top-level fields in the row are enumerated. Nested + /// child instances can be access through the method + /// to process nested content. + /// + public RowReader(ref RowBuffer row) + : this(ref row, RowCursor.Create(ref row)) + { + } + + public RowReader(ref RowBuffer row, in Checkpoint checkpoint) + { + this.row = row; + this.columns = checkpoint.Cursor.layout.Columns; + this.schematizedCount = checkpoint.Cursor.layout.NumFixed + checkpoint.Cursor.layout.NumVariable; + + this.state = checkpoint.State; + this.cursor = checkpoint.Cursor; + this.columnIndex = checkpoint.ColumnIndex; + } + + /// The current traversal state of the reader. + internal enum States : byte + { + /// The reader has not be started yet. + None, + + /// Enumerating schematized fields (fixed and variable) from left to right. + Schematized, + + /// Enumerating top-level fields of the current scope. + Sparse, + + /// The reader has completed the scope. + Done, + } + + /// The length of row in bytes. + public int Length => this.row.Length; + + /// The storage placement of the field (if positioned on a field, undefined otherwise). + public StorageKind Storage + { + get + { + switch (this.state) + { + case States.Schematized: + return this.columns[this.columnIndex].Storage; + case States.Sparse: + return StorageKind.Sparse; + default: + return default; + } + } + } + + /// The type of the field (if positioned on a field, undefined otherwise). + public LayoutType Type + { + get + { + switch (this.state) + { + case States.Schematized: + return this.columns[this.columnIndex].Type; + case States.Sparse: + return this.cursor.cellType; + default: + return default; + } + } + } + + /// The type arguments of the field (if positioned on a field, undefined otherwise). + public TypeArgumentList TypeArgs + { + get + { + switch (this.state) + { + case States.Schematized: + return this.columns[this.columnIndex].TypeArgs; + case States.Sparse: + return this.cursor.cellTypeArgs; + default: + return TypeArgumentList.Empty; + } + } + } + + /// True if field has a value (if positioned on a field, undefined otherwise). + /// + /// If the current field is a Nullable scope, this method return true if the value is not + /// null. If the current field is a nullable Null primitive value, this method return true if the value + /// is set (even though its values is set to null). + /// + public bool HasValue + { + get + { + switch (this.state) + { + case States.Schematized: + return true; + case States.Sparse: + if (this.cursor.cellType is LayoutNullable) + { + RowCursor nullableScope = this.row.SparseIteratorReadScope(ref this.cursor, immutable: true); + return LayoutNullable.HasValue(ref this.row, ref nullableScope) == Result.Success; + } + + return true; + default: + return false; + } + } + } + + /// + /// The path, relative to the scope, of the field (if positioned on a field, undefined + /// otherwise). + /// + /// When enumerating an indexed scope, this value is always null (see ). + public UtfAnyString Path + { + get + { + switch (this.state) + { + case States.Schematized: + return this.columns[this.columnIndex].Path; + case States.Sparse: + if (this.cursor.pathOffset == default) + { + return default; + } + + Utf8Span span = this.row.ReadSparsePath(ref this.cursor); + return Utf8String.CopyFrom(span); + default: + return default; + } + } + } + + /// + /// The path, relative to the scope, of the field (if positioned on a field, undefined + /// otherwise). + /// + /// When enumerating an indexed scope, this value is always null (see ). + public Utf8Span PathSpan + { + get + { + switch (this.state) + { + case States.Schematized: + return this.columns[this.columnIndex].Path.Span; + case States.Sparse: + return this.row.ReadSparsePath(ref this.cursor); + default: + return default; + } + } + } + + /// + /// The 0-based index, relative to the start of the scope, of the field (if positioned on a + /// field, undefined otherwise). + /// + /// When enumerating a non-indexed scope, this value is always 0 (see ). + public int Index + { + get + { + switch (this.state) + { + case States.Schematized: + return 0; + case States.Sparse: + return this.cursor.index; + default: + return default; + } + } + } + + public Checkpoint SaveCheckpoint() + { + return new Checkpoint(this.state, this.columnIndex, this.cursor); + } + + /// Advances the reader to the next field. + /// True, if there is another field to be read, false otherwise. + public bool Read() + { + switch (this.state) + { + case States.None: + { + if (this.cursor.scopeType is LayoutUDT) + { + this.state = States.Schematized; + goto case States.Schematized; + } + + this.state = States.Sparse; + goto case States.Sparse; + } + + case States.Schematized: + { + this.columnIndex++; + if (this.columnIndex >= this.schematizedCount) + { + this.state = States.Sparse; + goto case States.Sparse; + } + + Contract.Assert(this.cursor.scopeType is LayoutUDT); + LayoutColumn col = this.columns[this.columnIndex]; + if (!this.row.ReadBit(this.cursor.start, col.NullBit)) + { + // Skip schematized values if they aren't present. + goto case States.Schematized; + } + + return true; + } + + case States.Sparse: + { + if (!this.cursor.MoveNext(ref this.row)) + { + this.state = States.Done; + goto case States.Done; + } + + return true; + } + + case States.Done: + { + return false; + } + } + + return false; + } + + /// Read the current field as a . + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadBool(out bool value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutBoolean)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseBool(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a null. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadNull(out NullValue value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutNull)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseNull(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 8-bit, signed integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadInt8(out sbyte value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutInt8)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseInt8(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 16-bit, signed integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadInt16(out short value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutInt16)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseInt16(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 32-bit, signed integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadInt32(out int value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutInt32)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseInt32(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 64-bit, signed integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadInt64(out long value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutInt64)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseInt64(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 8-bit, unsigned integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadUInt8(out byte value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutUInt8)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseUInt8(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 16-bit, unsigned integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadUInt16(out ushort value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutUInt16)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseUInt16(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 32-bit, unsigned integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadUInt32(out uint value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutUInt32)) + { + value = default(int); + return Result.TypeMismatch; + } + + value = this.row.ReadSparseUInt32(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 64-bit, unsigned integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadUInt64(out ulong value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutUInt64)) + { + value = default(long); + return Result.TypeMismatch; + } + + value = this.row.ReadSparseUInt64(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a variable length, 7-bit encoded, signed integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadVarInt(out long value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutVarInt)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseVarInt(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a variable length, 7-bit encoded, unsigned integer. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadVarUInt(out ulong value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutVarUInt)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseVarUInt(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 32-bit, IEEE-encoded floating point value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadFloat32(out float value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutFloat32)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseFloat32(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 64-bit, IEEE-encoded floating point value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadFloat64(out double value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutFloat64)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseFloat64(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length, 128-bit, IEEE-encoded floating point value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadFloat128(out Float128 value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutFloat128)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseFloat128(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadDecimal(out decimal value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutDecimal)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseDecimal(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadDateTime(out DateTime value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutDateTime)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseDateTime(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadUnixDateTime(out UnixDateTime value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutUnixDateTime)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseUnixDateTime(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadGuid(out Guid value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutGuid)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseGuid(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a fixed length value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadMongoDbObjectId(out MongoDbObjectId value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutMongoDbObjectId)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseMongoDbObjectId(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a variable length, UTF8 encoded, string value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadString(out string value) + { + Result r = this.ReadString(out Utf8Span span); + value = (r == Result.Success) ? span.ToString() : default; + return r; + } + + /// Read the current field as a variable length, UTF8 encoded, string value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadString(out Utf8String value) + { + Result r = this.ReadString(out Utf8Span span); + value = (r == Result.Success) ? Utf8String.CopyFrom(span) : default; + return r; + } + + /// Read the current field as a variable length, UTF8 encoded, string value. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadString(out Utf8Span value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutUtf8)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseString(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a variable length, sequence of bytes. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadBinary(out byte[] value) + { + Result r = this.ReadBinary(out ReadOnlySpan span); + value = (r == Result.Success) ? span.ToArray() : default; + return r; + } + + /// Read the current field as a variable length, sequence of bytes. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + public Result ReadBinary(out ReadOnlySpan value) + { + switch (this.state) + { + case States.Schematized: + return this.ReadPrimitiveValue(out value); + case States.Sparse: + if (!(this.cursor.cellType is LayoutBinary)) + { + value = default; + return Result.TypeMismatch; + } + + value = this.row.ReadSparseBinary(ref this.cursor); + return Result.Success; + default: + value = default; + return Result.Failure; + } + } + + /// Read the current field as a nested, structured, sparse scope. + /// + /// Child readers can be used to read all sparse scope types including typed and untyped + /// objects, arrays, tuples, set, and maps. + /// + /// Nested child readers are independent of their parent. + /// + public RowReader ReadScope() + { + RowCursor newScope = this.row.SparseIteratorReadScope(ref this.cursor, immutable: true); + return new RowReader(ref this.row, newScope); + } + + /// A function to reader content from a . + /// The type of the context value passed by the caller. + /// A forward-only cursor for writing content. + /// A context value provided by the caller. + /// The result. + public delegate Result ReaderFunc(ref RowReader reader, TContext context); + + /// Read the current field as a nested, structured, sparse scope. + /// + /// Child readers can be used to read all sparse scope types including typed and untyped + /// objects, arrays, tuples, set, and maps. + /// + public Result ReadScope(TContext context, ReaderFunc func) + { + RowCursor childScope = this.row.SparseIteratorReadScope(ref this.cursor, immutable: true); + RowReader nestedReader = new RowReader(ref this.row, childScope); + + Result result = func?.Invoke(ref nestedReader, context) ?? Result.Success; + if (result != Result.Success) + { + return result; + } + + this.cursor.Skip(ref this.row, ref nestedReader.cursor); + return Result.Success; + } + + /// + /// Advance a reader to the end of a child reader. The child reader is also advanced to the + /// end of its scope. + /// + /// + /// The reader must not have been advanced since the child reader was created with ReadScope. + /// This method can be used when the overload of that takes a + /// is not an option, such as when TContext is a ref struct. + /// + public Result SkipScope(ref RowReader nestedReader) + { + if (nestedReader.cursor.start != this.cursor.valueOffset) + { + return Result.Failure; + } + + this.cursor.Skip(ref this.row, ref nestedReader.cursor); + return Result.Success; + } + + /// Read a generic schematized field value via the scope's layout. + /// The expected type of the field. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + private Result ReadPrimitiveValue(out TValue value) + { + LayoutColumn col = this.columns[this.columnIndex]; + LayoutType t = this.columns[this.columnIndex].Type; + if (!(t is LayoutType)) + { + value = default; + return Result.TypeMismatch; + } + + switch (col?.Storage) + { + case StorageKind.Fixed: + return t.TypeAs>().ReadFixed(ref this.row, ref this.cursor, col, out value); + case StorageKind.Variable: + return t.TypeAs>().ReadVariable(ref this.row, ref this.cursor, col, out value); + default: + Contract.Assert(false); + value = default; + return Result.Failure; + } + } + + /// Read a generic schematized field value via the scope's layout. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + private Result ReadPrimitiveValue(out Utf8Span value) + { + LayoutColumn col = this.columns[this.columnIndex]; + LayoutType t = this.columns[this.columnIndex].Type; + if (!(t is ILayoutUtf8SpanReadable)) + { + value = default; + return Result.TypeMismatch; + } + + switch (col?.Storage) + { + case StorageKind.Fixed: + return t.TypeAs().ReadFixed(ref this.row, ref this.cursor, col, out value); + case StorageKind.Variable: + return t.TypeAs().ReadVariable(ref this.row, ref this.cursor, col, out value); + default: + Contract.Assert(false); + value = default; + return Result.Failure; + } + } + + /// Read a generic schematized field value via the scope's layout. + /// The sub-element type of the field. + /// On success, receives the value, undefined otherwise. + /// Success if the read is successful, an error code otherwise. + private Result ReadPrimitiveValue(out ReadOnlySpan value) + { + LayoutColumn col = this.columns[this.columnIndex]; + LayoutType t = this.columns[this.columnIndex].Type; + if (!(t is ILayoutSpanReadable)) + { + value = default; + return Result.TypeMismatch; + } + + switch (col?.Storage) + { + case StorageKind.Fixed: + return t.TypeAs>().ReadFixed(ref this.row, ref this.cursor, col, out value); + case StorageKind.Variable: + return t.TypeAs>().ReadVariable(ref this.row, ref this.cursor, col, out value); + default: + Contract.Assert(false); + value = default; + return Result.Failure; + } + } + + /// + /// An encapsulation of the current state of a that can be used to + /// recreate the in the same logical position. + /// + public readonly struct Checkpoint + { + internal readonly States State; + internal readonly int ColumnIndex; + internal readonly RowCursor Cursor; + + internal Checkpoint(States state, int columnIndex, RowCursor cursor) + { + this.State = state; + this.ColumnIndex = columnIndex; + this.Cursor = cursor; + } + } + } +} diff --git a/dotnet/src/HybridRow/IO/RowReaderExtensions.cs b/dotnet/src/HybridRow/IO/RowReaderExtensions.cs new file mode 100644 index 0000000..aa9a72c --- /dev/null +++ b/dotnet/src/HybridRow/IO/RowReaderExtensions.cs @@ -0,0 +1,79 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.IO +{ + using System.Collections.Generic; + + public static class RowReaderExtensions + { + /// A function to read content from a . + /// The type of the item to read. + /// A forward-only cursor for reading the item. + /// On success, the item read. + /// The result. + public delegate Result DeserializerFunc(ref RowReader reader, out TItem item); + + /// Read the current field as a nested, structured, sparse scope containing a linear collection of zero or more items. + /// The type of the items within the collection. + /// A forward-only cursor for reading the collection. + /// A function that reads one item from the collection. + /// On success, the collection of materialized items. + /// The result. + public static Result ReadList(this ref RowReader reader, DeserializerFunc deserializer, out List list) + { + // Pass the context as a struct by value to avoid allocations. + ListContext ctx = new ListContext + { + List = new List(), + Deserializer = deserializer, + }; + + // All lambda's here are static. + Result r = reader.ReadScope( + ctx, + (ref RowReader arrayReader, ListContext ctx1) => + { + while (arrayReader.Read()) + { + Result r2 = arrayReader.ReadScope( + ctx1, + (ref RowReader itemReader, ListContext ctx2) => + { + Result r3 = ctx2.Deserializer(ref itemReader, out TItem op); + if (r3 != Result.Success) + { + return r3; + } + + ctx2.List.Add(op); + return Result.Success; + }); + + if (r2 != Result.Success) + { + return r2; + } + } + + return Result.Success; + }); + + if (r != Result.Success) + { + list = default; + return r; + } + + list = ctx.List; + return Result.Success; + } + + private struct ListContext + { + public List List; + public DeserializerFunc Deserializer; + } + } +} diff --git a/dotnet/src/HybridRow/IO/RowWriter.cs b/dotnet/src/HybridRow/IO/RowWriter.cs new file mode 100644 index 0000000..8b511b0 --- /dev/null +++ b/dotnet/src/HybridRow/IO/RowWriter.cs @@ -0,0 +1,810 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.IO +{ + using System; + using System.Buffers; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public ref struct RowWriter + { + private RowBuffer row; + private RowCursor cursor; + + /// Initializes a new instance of the struct. + /// The row to be read. + /// The scope into which items should be written. + /// + /// A instance writes the fields of a given scope from left to right + /// in a forward only manner. If the root scope is provided then all top-level fields in the row can be + /// written. + /// + private RowWriter(ref RowBuffer row, ref RowCursor scope) + { + this.row = row; + this.cursor = scope; + } + + /// A function to write content into a . + /// The type of the context value passed by the caller. + /// A forward-only cursor for writing content. + /// The type of the current scope. + /// A context value provided by the caller. + /// The result. + public delegate Result WriterFunc(ref RowWriter writer, TypeArgument typeArg, TContext context); + + private delegate void AccessMethod(ref RowWriter writer, TValue value); + + private delegate void AccessReadOnlySpanMethod(ref RowWriter writer, ReadOnlySpan value); + + private delegate void AccessUtf8SpanMethod(ref RowWriter writer, Utf8Span value); + + /// The resolver for UDTs. + public LayoutResolver Resolver => this.row.Resolver; + + /// The length of row in bytes. + public int Length => this.row.Length; + + /// The active layout of the current writer scope. + public Layout Layout => this.cursor.layout; + + /// Write an entire row in a streaming left-to-right way. + /// The type of the context value to pass to . + /// The row to write. + /// A context value to pass to . + /// A function to write the entire row. + /// Success if the write is successful, an error code otherwise. + public static Result WriteBuffer(ref RowBuffer row, TContext context, WriterFunc func) + { + RowCursor scope = RowCursor.Create(ref row); + RowWriter writer = new RowWriter(ref row, ref scope); + TypeArgument typeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(scope.layout.SchemaId)); + Result result = func(ref writer, typeArg, context); + row = writer.row; + return result; + } + + /// Write a field as a . + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteBool(UtfAnyString path, bool value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Boolean, + (ref RowWriter w, bool v) => w.row.WriteSparseBool(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a . + /// The scope-relative path of the field to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteNull(UtfAnyString path) + { + return this.WritePrimitive( + path, + NullValue.Default, + LayoutType.Null, + (ref RowWriter w, NullValue v) => w.row.WriteSparseNull(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 8-bit, signed integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteInt8(UtfAnyString path, sbyte value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Int8, + (ref RowWriter w, sbyte v) => w.row.WriteSparseInt8(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 16-bit, signed integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteInt16(UtfAnyString path, short value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Int16, + (ref RowWriter w, short v) => w.row.WriteSparseInt16(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 32-bit, signed integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteInt32(UtfAnyString path, int value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Int32, + (ref RowWriter w, int v) => w.row.WriteSparseInt32(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 64-bit, signed integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteInt64(UtfAnyString path, long value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Int64, + (ref RowWriter w, long v) => w.row.WriteSparseInt64(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 8-bit, unsigned integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteUInt8(UtfAnyString path, byte value) + { + return this.WritePrimitive( + path, + value, + LayoutType.UInt8, + (ref RowWriter w, byte v) => w.row.WriteSparseUInt8(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 16-bit, unsigned integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteUInt16(UtfAnyString path, ushort value) + { + return this.WritePrimitive( + path, + value, + LayoutType.UInt16, + (ref RowWriter w, ushort v) => w.row.WriteSparseUInt16(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 32-bit, unsigned integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteUInt32(UtfAnyString path, uint value) + { + return this.WritePrimitive( + path, + value, + LayoutType.UInt32, + (ref RowWriter w, uint v) => w.row.WriteSparseUInt32(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 64-bit, unsigned integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteUInt64(UtfAnyString path, ulong value) + { + return this.WritePrimitive( + path, + value, + LayoutType.UInt64, + (ref RowWriter w, ulong v) => w.row.WriteSparseUInt64(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, 7-bit encoded, signed integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteVarInt(UtfAnyString path, long value) + { + return this.WritePrimitive( + path, + value, + LayoutType.VarInt, + (ref RowWriter w, long v) => w.row.WriteSparseVarInt(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, 7-bit encoded, unsigned integer. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteVarUInt(UtfAnyString path, ulong value) + { + return this.WritePrimitive( + path, + value, + LayoutType.VarUInt, + (ref RowWriter w, ulong v) => w.row.WriteSparseVarUInt(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 32-bit, IEEE-encoded floating point value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteFloat32(UtfAnyString path, float value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Float32, + (ref RowWriter w, float v) => w.row.WriteSparseFloat32(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 64-bit, IEEE-encoded floating point value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteFloat64(UtfAnyString path, double value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Float64, + (ref RowWriter w, double v) => w.row.WriteSparseFloat64(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length, 128-bit, IEEE-encoded floating point value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteFloat128(UtfAnyString path, Float128 value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Float128, + (ref RowWriter w, Float128 v) => w.row.WriteSparseFloat128(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteDecimal(UtfAnyString path, decimal value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Decimal, + (ref RowWriter w, decimal v) => w.row.WriteSparseDecimal(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteDateTime(UtfAnyString path, DateTime value) + { + return this.WritePrimitive( + path, + value, + LayoutType.DateTime, + (ref RowWriter w, DateTime v) => w.row.WriteSparseDateTime(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteUnixDateTime(UtfAnyString path, UnixDateTime value) + { + return this.WritePrimitive( + path, + value, + LayoutType.UnixDateTime, + (ref RowWriter w, UnixDateTime v) => w.row.WriteSparseUnixDateTime(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteGuid(UtfAnyString path, Guid value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Guid, + (ref RowWriter w, Guid v) => w.row.WriteSparseGuid(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a fixed length value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteMongoDbObjectId(UtfAnyString path, MongoDbObjectId value) + { + return this.WritePrimitive( + path, + value, + LayoutType.MongoDbObjectId, + (ref RowWriter w, MongoDbObjectId v) => w.row.WriteSparseMongoDbObjectId(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, UTF8 encoded, string value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteString(UtfAnyString path, string value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Utf8, + (ref RowWriter w, string v) => w.row.WriteSparseString(ref w.cursor, Utf8Span.TranscodeUtf16(v), UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, UTF8 encoded, string value. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteString(UtfAnyString path, Utf8Span value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Utf8, + (ref RowWriter w, Utf8Span v) => w.row.WriteSparseString(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, sequence of bytes. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteBinary(UtfAnyString path, byte[] value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Binary, + (ref RowWriter w, byte[] v) => w.row.WriteSparseBinary(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, sequence of bytes. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteBinary(UtfAnyString path, ReadOnlySpan value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Binary, + (ref RowWriter w, ReadOnlySpan v) => w.row.WriteSparseBinary(ref w.cursor, v, UpdateOptions.Upsert)); + } + + /// Write a field as a variable length, sequence of bytes. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + public Result WriteBinary(UtfAnyString path, ReadOnlySequence value) + { + return this.WritePrimitive( + path, + value, + LayoutType.Binary, + (ref RowWriter w, ReadOnlySequence v) => w.row.WriteSparseBinary(ref w.cursor, v, UpdateOptions.Upsert)); + } + + public Result WriteScope(UtfAnyString path, TypeArgument typeArg, TContext context, WriterFunc func) + { + LayoutType type = typeArg.Type; + Result result = this.PrepareSparseWrite(path, typeArg); + if (result != Result.Success) + { + return result; + } + + RowCursor nestedScope; + switch (type) + { + case LayoutObject scopeType: + this.row.WriteSparseObject(ref this.cursor, scopeType, UpdateOptions.Upsert, out nestedScope); + break; + case LayoutArray scopeType: + this.row.WriteSparseArray(ref this.cursor, scopeType, UpdateOptions.Upsert, out nestedScope); + break; + case LayoutTypedArray scopeType: + this.row.WriteTypedArray( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + case LayoutTuple scopeType: + this.row.WriteSparseTuple( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + case LayoutTypedTuple scopeType: + this.row.WriteTypedTuple( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + case LayoutTagged scopeType: + this.row.WriteTypedTuple( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + case LayoutTagged2 scopeType: + this.row.WriteTypedTuple( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + case LayoutNullable scopeType: + this.row.WriteNullable( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + func != null, + out nestedScope); + + break; + case LayoutUDT scopeType: + Layout udt = this.row.Resolver.Resolve(typeArg.TypeArgs.SchemaId); + this.row.WriteSparseUDT(ref this.cursor, scopeType, udt, UpdateOptions.Upsert, out nestedScope); + break; + + case LayoutTypedSet scopeType: + this.row.WriteTypedSet( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + case LayoutTypedMap scopeType: + this.row.WriteTypedMap( + ref this.cursor, + scopeType, + typeArg.TypeArgs, + UpdateOptions.Upsert, + out nestedScope); + + break; + + default: + return Result.Failure; + } + + RowWriter nestedWriter = new RowWriter(ref this.row, ref nestedScope); + result = func?.Invoke(ref nestedWriter, typeArg, context) ?? Result.Success; + this.row = nestedWriter.row; + nestedScope.count = nestedWriter.cursor.count; + + if (result != Result.Success) + { + // TODO: what about unique violations here? + return result; + } + + if (type is LayoutUniqueScope) + { + result = this.row.TypedCollectionUniqueIndexRebuild(ref nestedScope); + if (result != Result.Success) + { + // TODO: If the index rebuild fails then the row is corrupted. Should we automatically clean up here? + return result; + } + } + + this.cursor.MoveNext(ref this.row, ref nestedWriter.cursor); + return Result.Success; + } + + /// Helper for writing a primitive value. + /// The type of the primitive value. + /// The scope-relative path of the field to write. + /// The value to write. + /// The layout type. + /// The access method for . + /// Success if the write is successful, an error code otherwise. + private Result WritePrimitive(UtfAnyString path, TValue value, LayoutType type, AccessMethod sparse) + { + Result result = Result.NotFound; + if (this.cursor.scopeType is LayoutUDT) + { + result = this.WriteSchematizedValue(path, value); + } + + if (result == Result.NotFound) + { + // Write sparse value. + + result = this.PrepareSparseWrite(path, type.TypeArg); + if (result != Result.Success) + { + return result; + } + + sparse(ref this, value); + this.cursor.MoveNext(ref this.row); + } + + return result; + } + + /// Helper for writing a primitive value. + /// The type of layout type. + /// The scope-relative path of the field to write. + /// The value to write. + /// The layout type. + /// The access method for . + /// Success if the write is successful, an error code otherwise. + private Result WritePrimitive( + UtfAnyString path, + Utf8Span value, + TLayoutType type, + AccessUtf8SpanMethod sparse) + where TLayoutType : LayoutType, ILayoutUtf8SpanWritable + { + Result result = Result.NotFound; + if (this.cursor.scopeType is LayoutUDT) + { + result = this.WriteSchematizedValue(path, value); + } + + if (result == Result.NotFound) + { + // Write sparse value. + result = this.PrepareSparseWrite(path, type.TypeArg); + if (result != Result.Success) + { + return result; + } + + sparse(ref this, value); + this.cursor.MoveNext(ref this.row); + } + + return result; + } + + /// Helper for writing a primitive value. + /// The type of layout type. + /// The sub-element type of the field. + /// The scope-relative path of the field to write. + /// The value to write. + /// The layout type. + /// The access method for . + /// Success if the write is successful, an error code otherwise. + private Result WritePrimitive( + UtfAnyString path, + ReadOnlySpan value, + TLayoutType type, + AccessReadOnlySpanMethod sparse) + where TLayoutType : LayoutType, ILayoutSpanWritable + { + Result result = Result.NotFound; + if (this.cursor.scopeType is LayoutUDT) + { + result = this.WriteSchematizedValue(path, value); + } + + if (result == Result.NotFound) + { + // Write sparse value. + result = this.PrepareSparseWrite(path, type.TypeArg); + if (result != Result.Success) + { + return result; + } + + sparse(ref this, value); + this.cursor.MoveNext(ref this.row); + } + + return result; + } + + /// Helper for writing a primitive value. + /// The type of layout type. + /// The sub-element type of the field. + /// The scope-relative path of the field to write. + /// The value to write. + /// The layout type. + /// The access method for . + /// Success if the write is successful, an error code otherwise. + private Result WritePrimitive( + UtfAnyString path, + ReadOnlySequence value, + TLayoutType type, + AccessMethod> sparse) + where TLayoutType : LayoutType, ILayoutSequenceWritable + { + Result result = Result.NotFound; + if (this.cursor.scopeType is LayoutUDT) + { + result = this.WriteSchematizedValue(path, value); + } + + if (result == Result.NotFound) + { + // Write sparse value. + result = this.PrepareSparseWrite(path, type.TypeArg); + if (result != Result.Success) + { + return result; + } + + sparse(ref this, value); + this.cursor.MoveNext(ref this.row); + } + + return result; + } + + /// Helper for preparing the write of a sparse field. + /// The path identifying the field to write. + /// The (optional) type constraints. + /// Success if the write is permitted, the error code otherwise. + private Result PrepareSparseWrite(UtfAnyString path, TypeArgument typeArg) + { + if (this.cursor.scopeType.IsFixedArity && !(this.cursor.scopeType is LayoutNullable)) + { + if ((this.cursor.index < this.cursor.scopeTypeArgs.Count) && !typeArg.Equals(this.cursor.scopeTypeArgs[this.cursor.index])) + { + return Result.TypeConstraint; + } + } + else if (this.cursor.scopeType is LayoutTypedMap) + { + if (!typeArg.Equals(this.cursor.scopeType.TypeAs().FieldType(ref this.cursor))) + { + return Result.TypeConstraint; + } + } + else if (this.cursor.scopeType.IsTypedScope && !typeArg.Equals(this.cursor.scopeTypeArgs[0])) + { + return Result.TypeConstraint; + } + + this.cursor.writePath = path; + return Result.Success; + } + + /// Write a generic schematized field value via the scope's layout. + /// The expected type of the field. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + private Result WriteSchematizedValue(UtfAnyString path, TValue value) + { + if (!this.cursor.layout.TryFind(path, out LayoutColumn col)) + { + return Result.NotFound; + } + + if (!(col.Type is LayoutType t)) + { + return Result.NotFound; + } + + switch (col.Storage) + { + case StorageKind.Fixed: + return t.WriteFixed(ref this.row, ref this.cursor, col, value); + + case StorageKind.Variable: + return t.WriteVariable(ref this.row, ref this.cursor, col, value); + + default: + return Result.NotFound; + } + + return Result.NotFound; + } + + /// Write a generic schematized field value via the scope's layout. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + private Result WriteSchematizedValue(UtfAnyString path, Utf8Span value) + { + if (!this.cursor.layout.TryFind(path, out LayoutColumn col)) + { + return Result.NotFound; + } + + LayoutType t = col.Type; + if (!(t is ILayoutUtf8SpanWritable)) + { + return Result.NotFound; + } + + switch (col.Storage) + { + case StorageKind.Fixed: + return t.TypeAs().WriteFixed(ref this.row, ref this.cursor, col, value); + case StorageKind.Variable: + return t.TypeAs().WriteVariable(ref this.row, ref this.cursor, col, value); + default: + return Result.NotFound; + } + } + + /// Write a generic schematized field value via the scope's layout. + /// The sub-element type of the field. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + private Result WriteSchematizedValue(UtfAnyString path, ReadOnlySpan value) + { + if (!this.cursor.layout.TryFind(path, out LayoutColumn col)) + { + return Result.NotFound; + } + + LayoutType t = col.Type; + if (!(t is ILayoutSpanWritable)) + { + return Result.NotFound; + } + + switch (col.Storage) + { + case StorageKind.Fixed: + return t.TypeAs>().WriteFixed(ref this.row, ref this.cursor, col, value); + case StorageKind.Variable: + return t.TypeAs>().WriteVariable(ref this.row, ref this.cursor, col, value); + default: + return Result.NotFound; + } + } + + /// Write a generic schematized field value via the scope's layout. + /// The sub-element type of the field. + /// The scope-relative path of the field to write. + /// The value to write. + /// Success if the write is successful, an error code otherwise. + private Result WriteSchematizedValue(UtfAnyString path, ReadOnlySequence value) + { + if (!this.cursor.layout.TryFind(path, out LayoutColumn col)) + { + return Result.NotFound; + } + + LayoutType t = col.Type; + if (!(t is ILayoutSequenceWritable)) + { + return Result.NotFound; + } + + switch (col.Storage) + { + case StorageKind.Fixed: + return t.TypeAs>().WriteFixed(ref this.row, ref this.cursor, col, value); + case StorageKind.Variable: + return t.TypeAs>().WriteVariable(ref this.row, ref this.cursor, col, value); + default: + return Result.NotFound; + } + } + } +} diff --git a/dotnet/src/HybridRow/ISpanResizer.cs b/dotnet/src/HybridRow/ISpanResizer.cs new file mode 100644 index 0000000..c9a8670 --- /dev/null +++ b/dotnet/src/HybridRow/ISpanResizer.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + + public interface ISpanResizer + { + /// Resizes an existing a buffer. + /// The type of the elements of the memory. + /// The minimum required length (in elements) of the memory. + /// + /// Optional existing memory to be copied to the new buffer. Ownership of is + /// transferred as part of this call and it should not be used by the caller after this call completes. + /// + /// + /// A new memory whose size is at least as big as + /// and containing the content of . + /// + Span Resize(int minimumLength, Span buffer = default); + } +} diff --git a/dotnet/src/HybridRow/Internal/MurmurHash3.cs b/dotnet/src/HybridRow/Internal/MurmurHash3.cs new file mode 100644 index 0000000..78f8d71 --- /dev/null +++ b/dotnet/src/HybridRow/Internal/MurmurHash3.cs @@ -0,0 +1,229 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Internal +{ + using System; + using System.Runtime.CompilerServices; + using System.Runtime.InteropServices; + using System.Text; + using Microsoft.Azure.Cosmos.Core; + + /// + /// MurmurHash3 for x64 (Little Endian). + ///

Reference: https://en.wikipedia.org/wiki/MurmurHash

+ ///

+ /// This implementation provides span-based access for hashing content not available in a + /// + ///

+ /// + internal static class MurmurHash3 + { + /// MurmurHash3 128-bit implementation. + /// The data to hash. + /// The seed to initialize with. + /// The 128-bit hash represented as two 64-bit words. + public static (ulong low, ulong high) Hash128(string value, (ulong high, ulong low) seed) + { + Contract.Requires(value != null); + int size = Encoding.UTF8.GetMaxByteCount(value.Length); + Span span = size <= 256 ? stackalloc byte[size] : new byte[size]; + int len = Encoding.UTF8.GetBytes(value.AsSpan(), span); + return MurmurHash3.Hash128(span.Slice(0, len), seed); + } + + /// MurmurHash3 128-bit implementation. + /// The data to hash. + /// The seed to initialize with. + /// The 128-bit hash represented as two 64-bit words. + public static (ulong low, ulong high) Hash128(bool value, (ulong high, ulong low) seed) + { + // Ensure that a bool is ALWAYS a single byte encoding with 1 for true and 0 for false. + return MurmurHash3.Hash128((byte)(value ? 1 : 0), seed); + } + + /// MurmurHash3 128-bit implementation. + /// The data to hash. + /// The seed to initialize with. + /// The 128-bit hash represented as two 64-bit words. + public static unsafe (ulong low, ulong high) Hash128(T value, (ulong high, ulong low) seed) + where T : unmanaged + { + ReadOnlySpan span = new ReadOnlySpan(&value, 1); + return MurmurHash3.Hash128(MemoryMarshal.AsBytes(span), seed); + } + + /// MurmurHash3 128-bit implementation. + /// The data to hash. + /// The seed to initialize with. + /// The 128-bit hash represented as two 64-bit words. + public static unsafe (ulong low, ulong high) Hash128(ReadOnlySpan span, (ulong high, ulong low) seed) + { + Contract.Assert(BitConverter.IsLittleEndian, "Little Endian expected"); + const ulong c1 = 0x87c37b91114253d5; + const ulong c2 = 0x4cf5ad432745937f; + + (ulong h1, ulong h2) = seed; + + // body + unchecked + { + fixed (byte* words = span) + { + int position; + for (position = 0; position < span.Length - 15; position += 16) + { + ulong k1 = *(ulong*)(words + position); + ulong k2 = *(ulong*)(words + position + 8); + + // k1, h1 + k1 *= c1; + k1 = MurmurHash3.RotateLeft64(k1, 31); + k1 *= c2; + + h1 ^= k1; + h1 = MurmurHash3.RotateLeft64(h1, 27); + h1 += h2; + h1 = (h1 * 5) + 0x52dce729; + + // k2, h2 + k2 *= c2; + k2 = MurmurHash3.RotateLeft64(k2, 33); + k2 *= c1; + + h2 ^= k2; + h2 = MurmurHash3.RotateLeft64(h2, 31); + h2 += h1; + h2 = (h2 * 5) + 0x38495ab5; + } + + { + // tail + ulong k1 = 0; + ulong k2 = 0; + + int n = span.Length & 15; + if (n >= 15) + { + k2 ^= (ulong)words[position + 14] << 48; + } + + if (n >= 14) + { + k2 ^= (ulong)words[position + 13] << 40; + } + + if (n >= 13) + { + k2 ^= (ulong)words[position + 12] << 32; + } + + if (n >= 12) + { + k2 ^= (ulong)words[position + 11] << 24; + } + + if (n >= 11) + { + k2 ^= (ulong)words[position + 10] << 16; + } + + if (n >= 10) + { + k2 ^= (ulong)words[position + 09] << 8; + } + + if (n >= 9) + { + k2 ^= (ulong)words[position + 08] << 0; + } + + k2 *= c2; + k2 = MurmurHash3.RotateLeft64(k2, 33); + k2 *= c1; + h2 ^= k2; + + if (n >= 8) + { + k1 ^= (ulong)words[position + 7] << 56; + } + + if (n >= 7) + { + k1 ^= (ulong)words[position + 6] << 48; + } + + if (n >= 6) + { + k1 ^= (ulong)words[position + 5] << 40; + } + + if (n >= 5) + { + k1 ^= (ulong)words[position + 4] << 32; + } + + if (n >= 4) + { + k1 ^= (ulong)words[position + 3] << 24; + } + + if (n >= 3) + { + k1 ^= (ulong)words[position + 2] << 16; + } + + if (n >= 2) + { + k1 ^= (ulong)words[position + 1] << 8; + } + + if (n >= 1) + { + k1 ^= (ulong)words[position + 0] << 0; + } + + k1 *= c1; + k1 = MurmurHash3.RotateLeft64(k1, 31); + k1 *= c2; + h1 ^= k1; + } + } + + // finalization + h1 ^= (ulong)span.Length; + h2 ^= (ulong)span.Length; + h1 += h2; + h2 += h1; + h1 = MurmurHash3.Mix(h1); + h2 = MurmurHash3.Mix(h2); + h1 += h2; + h2 += h1; + } + + return (h1, h2); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Mix(ulong h) + { + unchecked + { + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + h ^= h >> 33; + return h; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong RotateLeft64(ulong n, int numBits) + { + Contract.Assert(numBits < 64, "numBits < 64"); + return (n << numBits) | (n >> (64 - numBits)); + } + } +} diff --git a/dotnet/src/HybridRow/Internal/Utf8StringJsonConverter.cs b/dotnet/src/HybridRow/Internal/Utf8StringJsonConverter.cs new file mode 100644 index 0000000..ca6473b --- /dev/null +++ b/dotnet/src/HybridRow/Internal/Utf8StringJsonConverter.cs @@ -0,0 +1,29 @@ +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Internal +{ + using System; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Newtonsoft.Json; + + /// Helper class for parsing from JSON. + internal class Utf8StringJsonConverter : JsonConverter + { + public override bool CanWrite => true; + + public override bool CanConvert(Type objectType) + { + return typeof(Utf8String).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + Contract.Requires(reader.TokenType == JsonToken.String); + return Utf8String.TranscodeUtf16((string)reader.Value); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + writer.WriteValue(((Utf8String)value).ToString()); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/Layout.cs b/dotnet/src/HybridRow/Layouts/Layout.cs new file mode 100644 index 0000000..890f04b --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/Layout.cs @@ -0,0 +1,175 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.CompilerServices; + using System.Text; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + /// A Layout describes the structure of a Hybrd Row. + /// + /// A layout indicates the number, order, and type of all schematized columns to be stored + /// within a hybrid row. The order and type of columns defines the physical ordering of bytes used to + /// encode the row and impacts the cost of updating the row. + /// + /// A layout is created by compiling a through or + /// by constructor through a . + /// + /// is immutable. + /// + public sealed class Layout + { + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly Layout Empty = SystemSchema.LayoutResolver.Resolve(SystemSchema.EmptySchemaId); + + private readonly LayoutColumn[] topColumns; + private readonly Dictionary pathMap; + private readonly Dictionary pathStringMap; + + internal Layout(string name, SchemaId schemaId, int numBitmaskBytes, int minRequiredSize, List columns) + { + this.Name = name; + this.SchemaId = schemaId; + this.NumBitmaskBytes = numBitmaskBytes; + this.Size = minRequiredSize; + this.Tokenizer = new StringTokenizer(); + this.pathMap = new Dictionary(columns.Count, SamplingUtf8StringComparer.Default); + this.pathStringMap = new Dictionary(columns.Count); + this.NumFixed = 0; + this.NumVariable = 0; + + List top = new List(columns.Count); + foreach (LayoutColumn c in columns) + { + this.Tokenizer.Add(c.Path); + this.pathMap.Add(c.FullPath, c); + this.pathStringMap.Add(c.FullPath.ToString(), c); + if (c.Storage == StorageKind.Fixed) + { + this.NumFixed++; + } + else if (c.Storage == StorageKind.Variable) + { + this.NumVariable++; + } + + if (c.Parent == null) + { + top.Add(c); + } + } + + this.topColumns = top.ToArray(); + } + + /// Name of the layout. + /// + /// Usually this is the name of the from which this + /// was generated. + /// + public string Name { get; } + + /// Unique identifier of the schema from which this was generated. + public SchemaId SchemaId { get; } + + /// The set of top level columns defined in the layout (in left-to-right order). + public ReadOnlySpan Columns => this.topColumns.AsSpan(); + + /// Minimum required size of a row of this layout. + /// + /// This size excludes all sparse columns, and assumes all columns (including variable) are + /// null. + /// + public int Size { get; } + + /// The number of bitmask bytes allocated within the layout. + /// + /// A presence bit is allocated for each fixed and variable-length field. Sparse columns + /// never have presence bits. Fixed boolean allocate an additional bit from the bitmask to store their + /// value. + /// + public int NumBitmaskBytes { get; } + + /// The number of fixed columns. + public int NumFixed { get; } + + /// The number of variable-length columns. + public int NumVariable { get; } + + /// A tokenizer for path strings. + [SuppressMessage( + "Microsoft.Performance", + "CA1822:MarkMembersAsStatic", + Justification = "Bug in Analyzer. This is an auto-property not a method.")] + public StringTokenizer Tokenizer + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get; + } + + /// Finds a column specification for a column with a matching path. + /// The path of the column to find. + /// If found, the column specification, otherwise null. + /// True if a column with the path is found, otherwise false. + public bool TryFind(UtfAnyString path, out LayoutColumn column) + { + if (path.IsNull) + { + column = default; + return false; + } + + if (path.IsUtf8) + { + return this.pathMap.TryGetValue(path.ToUtf8String(), out column); + } + + return this.pathStringMap.TryGetValue(path, out column); + } + + /// Finds a column specification for a column with a matching path. + /// The path of the column to find. + /// If found, the column specification, otherwise null. + /// True if a column with the path is found, otherwise false. + public bool TryFind(string path, out LayoutColumn column) + { + return this.pathStringMap.TryGetValue(path, out column); + } + + /// Returns a human readable diagnostic string representation of this . + /// This representation should only be used for debugging and diagnostic purposes. + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + sb.AppendFormat("Layout:\n"); + sb.AppendFormat("\tCount: {0}\n", this.topColumns.Length); + sb.AppendFormat("\tFixedSize: {0}\n", this.Size); + foreach (LayoutColumn c in this.topColumns) + { + if (c.Type.IsFixed) + { + if (c.Type.IsBool) + { + sb.AppendFormat("\t{0}: {1} @ {2}:{3}:{4}\n", c.FullPath, c.Type.Name, c.Offset, c.NullBit, c.BoolBit); + } + else + { + sb.AppendFormat("\t{0}: {1} @ {2}\n", c.FullPath, c.Type.Name, c.Offset); + } + } + else + { + sb.AppendFormat("\t{0}: {1}[{3}] @ {2}\n", c.FullPath, c.Type.Name, c.Offset, c.Size); + } + } + + return sb.ToString(); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutBit.cs b/dotnet/src/HybridRow/Layouts/LayoutBit.cs new file mode 100644 index 0000000..b4416f3 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutBit.cs @@ -0,0 +1,123 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Core; + + public readonly struct LayoutBit : IEquatable + { + /// The empty bit. + public static readonly LayoutBit Invalid = new LayoutBit(-1); + + /// The 0-based offset into the layout bitmask. + private readonly int index; + + /// Initializes a new instance of the struct. + /// The 0-based offset into the layout bitmask. + internal LayoutBit(int index) + { + Contract.Requires(index >= -1); + this.index = index; + } + + /// The 0-based offset into the layout bitmask. + public bool IsInvalid + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.index == -1; + } + + /// The 0-based offset into the layout bitmask. + public int Index + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.index; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool operator ==(LayoutBit left, LayoutBit right) + { + return left.Equals(right); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool operator !=(LayoutBit left, LayoutBit right) + { + return !(left == right); + } + + /// + /// Returns the 0-based byte offset from the beginning of the row or scope that contains the + /// bit from the bitmask. + /// + /// Also see to identify. + /// The byte offset from the beginning of the row where the scope begins. + /// The byte offset containing this bit. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetOffset(int offset) + { + return offset + (this.index / LayoutType.BitsPerByte); + } + + /// Returns the 0-based bit from the beginning of the byte that contains this bit. + /// Also see to identify relevant byte. + /// The bit of the byte within the bitmask. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetBit() + { + return this.index % LayoutType.BitsPerByte; + } + + public override bool Equals(object other) + { + return other is LayoutBit && this.Equals((LayoutBit)other); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Equals(LayoutBit other) + { + return this.index == other.index; + } + + public override int GetHashCode() + { + return this.index.GetHashCode(); + } + + /// Compute the division rounding up to the next whole number. + /// The numerator to divide. + /// The divisor to divide by. + /// The ceiling(numerator/divisor). + internal static int DivCeiling(int numerator, int divisor) + { + return (numerator + (divisor - 1)) / divisor; + } + + /// Allocates layout bits from a bitmask. + internal class Allocator + { + /// The next bit to allocate. + private int next; + + /// Initializes a new instance of the class. + public Allocator() + { + this.next = 0; + } + + /// The number of bytes needed to hold all bits so far allocated. + public int NumBytes => LayoutBit.DivCeiling(this.next, LayoutType.BitsPerByte); + + /// Allocates a new bit from the bitmask. + /// The allocated bit. + public LayoutBit Allocate() + { + return new LayoutBit(this.next++); + } + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutBuilder.cs b/dotnet/src/HybridRow/Layouts/LayoutBuilder.cs new file mode 100644 index 0000000..188ff15 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutBuilder.cs @@ -0,0 +1,230 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + internal sealed class LayoutBuilder + { + private readonly string name; + private readonly SchemaId schemaId; + private LayoutBit.Allocator bitallocator; + private List fixedColumns; + private int fixedCount; + private int fixedSize; + private List varColumns; + private int varCount; + private List sparseColumns; + private int sparseCount; + private Stack scope; + + // [ + // + // ... + // ... + // ... + // ] + public LayoutBuilder(string name, SchemaId schemaId) + { + this.name = name; + this.schemaId = schemaId; + this.Reset(); + } + + private LayoutColumn Parent + { + get + { + if (this.scope.Count == 0) + { + return null; + } + + return this.scope.Peek(); + } + } + + public void AddFixedColumn(string path, LayoutType type, bool nullable, int length = 0) + { + Contract.Requires(length >= 0); + Contract.Requires(!type.IsVarint); + + LayoutColumn col; + if (type.IsNull) + { + Contract.Requires(nullable); + LayoutBit nullbit = this.bitallocator.Allocate(); + col = new LayoutColumn( + path, + type, + TypeArgumentList.Empty, + StorageKind.Fixed, + this.Parent, + this.fixedCount, + 0, + nullbit, + boolBit: LayoutBit.Invalid); + } + else if (type.IsBool) + { + LayoutBit nullbit = nullable ? this.bitallocator.Allocate() : LayoutBit.Invalid; + LayoutBit boolbit = this.bitallocator.Allocate(); + col = new LayoutColumn( + path, + type, + TypeArgumentList.Empty, + StorageKind.Fixed, + this.Parent, + this.fixedCount, + 0, + nullbit, + boolBit: boolbit); + } + else + { + LayoutBit nullbit = nullable ? this.bitallocator.Allocate() : LayoutBit.Invalid; + col = new LayoutColumn( + path, + type, + TypeArgumentList.Empty, + StorageKind.Fixed, + this.Parent, + this.fixedCount, + this.fixedSize, + nullbit, + boolBit: LayoutBit.Invalid, + length: length); + + this.fixedSize += type.IsFixed ? type.Size : length; + } + + this.fixedCount++; + this.fixedColumns.Add(col); + } + + public void AddVariableColumn(string path, LayoutType type, int length = 0) + { + Contract.Requires(length >= 0); + Contract.Requires(type.AllowVariable); + + LayoutColumn col = new LayoutColumn( + path, + type, + TypeArgumentList.Empty, + StorageKind.Variable, + this.Parent, + this.varCount, + this.varCount, + nullBit: this.bitallocator.Allocate(), + boolBit: LayoutBit.Invalid, + length: length); + + this.varCount++; + this.varColumns.Add(col); + } + + public void AddSparseColumn(string path, LayoutType type) + { + LayoutColumn col = new LayoutColumn( + path, + type, + TypeArgumentList.Empty, + StorageKind.Sparse, + this.Parent, + this.sparseCount, + -1, + nullBit: LayoutBit.Invalid, + boolBit: LayoutBit.Invalid); + + this.sparseCount++; + this.sparseColumns.Add(col); + } + + public void AddObjectScope(string path, LayoutType type) + { + LayoutColumn col = new LayoutColumn( + path, + type, + TypeArgumentList.Empty, + StorageKind.Sparse, + this.Parent, + this.sparseCount, + -1, + nullBit: LayoutBit.Invalid, + boolBit: LayoutBit.Invalid); + + this.sparseCount++; + this.sparseColumns.Add(col); + this.scope.Push(col); + } + + public void EndObjectScope() + { + Contract.Requires(this.scope.Count > 0); + this.scope.Pop(); + } + + public void AddTypedScope(string path, LayoutType type, TypeArgumentList typeArgs) + { + LayoutColumn col = new LayoutColumn( + path, + type, + typeArgs, + StorageKind.Sparse, + this.Parent, + this.sparseCount, + -1, + nullBit: LayoutBit.Invalid, + boolBit: LayoutBit.Invalid); + + this.sparseCount++; + this.sparseColumns.Add(col); + } + + public Layout Build() + { + // Compute offset deltas. Offset bools by the present byte count, and fixed fields by the sum of the present and bool count. + int fixedDelta = this.bitallocator.NumBytes; + int varIndexDelta = this.fixedCount; + + // Update the fixedColumns with the delta before freezing them. + List updatedColumns = new List(this.fixedColumns.Count + this.varColumns.Count); + + foreach (LayoutColumn c in this.fixedColumns) + { + c.SetOffset(c.Offset + fixedDelta); + updatedColumns.Add(c); + } + + foreach (LayoutColumn c in this.varColumns) + { + // Adjust variable column indexes such that they begin immediately following the last fixed column. + c.SetIndex(c.Index + varIndexDelta); + updatedColumns.Add(c); + } + + updatedColumns.AddRange(this.sparseColumns); + + Layout layout = new Layout(this.name, this.schemaId, this.bitallocator.NumBytes, this.fixedSize + fixedDelta, updatedColumns); + this.Reset(); + return layout; + } + + private void Reset() + { + this.bitallocator = new LayoutBit.Allocator(); + this.fixedSize = 0; + this.fixedCount = 0; + this.fixedColumns = new List(); + this.varCount = 0; + this.varColumns = new List(); + this.sparseCount = 0; + this.sparseColumns = new List(); + this.scope = new Stack(); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutCode.cs b/dotnet/src/HybridRow/Layouts/LayoutCode.cs new file mode 100644 index 0000000..ea01de3 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutCode.cs @@ -0,0 +1,74 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1028 // Enum Storage should be Int32 + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + /// Type coded used in the binary encoding to indicate the formatting of succeeding bytes. + public enum LayoutCode : byte + { + Invalid = 0, + + Null = 1, + BooleanFalse = 2, + Boolean = 3, + + Int8 = 5, + Int16 = 6, + Int32 = 7, + Int64 = 8, + UInt8 = 9, + UInt16 = 10, + UInt32 = 11, + UInt64 = 12, + VarInt = 13, + VarUInt = 14, + + Float32 = 15, + Float64 = 16, + Decimal = 17, + + DateTime = 18, + Guid = 19, + + Utf8 = 20, + Binary = 21, + + Float128 = 22, + UnixDateTime = 23, + MongoDbObjectId = 24, + + ObjectScope = 30, + ImmutableObjectScope = 31, + ArrayScope = 32, + ImmutableArrayScope = 33, + TypedArrayScope = 34, + ImmutableTypedArrayScope = 35, + TupleScope = 36, + ImmutableTupleScope = 37, + TypedTupleScope = 38, + ImmutableTypedTupleScope = 39, + MapScope = 40, + ImmutableMapScope = 41, + TypedMapScope = 42, + ImmutableTypedMapScope = 43, + SetScope = 44, + ImmutableSetScope = 45, + TypedSetScope = 46, + ImmutableTypedSetScope = 47, + NullableScope = 48, + ImmutableNullableScope = 49, + TaggedScope = 50, + ImmutableTaggedScope = 51, + Tagged2Scope = 52, + ImmutableTagged2Scope = 53, + + /// Nested row. + Schema = 68, + ImmutableSchema = 69, + + EndScope = 70, + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutCodeTraits.cs b/dotnet/src/HybridRow/Layouts/LayoutCodeTraits.cs new file mode 100644 index 0000000..0f2ca5a --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutCodeTraits.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System.Runtime.CompilerServices; + + internal static class LayoutCodeTraits + { + /// Returns the same scope code without the immutable bit set. + /// The scope type code. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static LayoutCode ClearImmutableBit(LayoutCode code) + { + return code & (LayoutCode)0xFE; + } + + /// + /// Returns true if the type code indicates that, even within a typed scope, this element type + /// always requires a type code (because the value itself is in the type code). + /// + /// The element type code. + internal static bool AlwaysRequiresTypeCode(LayoutCode code) + { + return (code == LayoutCode.Boolean) || + (code == LayoutCode.BooleanFalse) || + (code == LayoutCode.Null); + } + + /// Returns a canonicalized version of the layout code. + /// + /// Some codes (e.g. use multiple type codes to also encode + /// values. This function converts actual value based code into the canonicalized type code for schema + /// comparisons. + /// + /// The code to canonicalize. + internal static LayoutCode Canonicalize(LayoutCode code) + { + return (code == LayoutCode.BooleanFalse) ? LayoutCode.Boolean : code; + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutColumn.cs b/dotnet/src/HybridRow/Layouts/LayoutColumn.cs new file mode 100644 index 0000000..ce7291c --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutColumn.cs @@ -0,0 +1,228 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System.Diagnostics; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + [DebuggerDisplay("{FullPath + \": \" + Type.Name + TypeArgs.ToString()}")] + public sealed class LayoutColumn + { + /// + /// If then the 0-based extra index within the bool byte + /// holding the value of this type, otherwise must be 0. + /// + private readonly int size; + + /// The relative path of the field within its parent scope. + private readonly Utf8String path; + + /// The full logical path of the field within the row. + private readonly Utf8String fullPath; + + /// The physical layout type of the field. + private readonly LayoutType type; + + /// The physical layout type of the field. + private readonly TypeArgument typeArg; + + /// The storage kind of the field. + private readonly StorageKind storage; + + /// The layout of the parent scope, if a nested column, otherwise null. + private readonly LayoutColumn parent; + + /// For types with generic parameters (e.g. , the type parameters. + private readonly TypeArgumentList typeArgs; + + /// For nullable fields, the 0-based index into the bit mask for the null bit. + private readonly LayoutBit nullBit; + + /// For bool fields, 0-based index into the bit mask for the bool value. + private readonly LayoutBit boolBit; + + /// + /// 0-based index of the column within the structure. Also indicates which presence bit + /// controls this column. + /// + private int index; + + /// + /// If equals then the byte offset to + /// the field location. + /// + /// If equals then the 0-based index of the + /// field from the beginning of the variable length segment. + /// + /// For all other values of , is ignored. + /// + private int offset; + + /// Initializes a new instance of the class. + /// The path to the field relative to parent scope. + /// Type of the field. + /// Storage encoding of the field. + /// The layout of the parent scope, if a nested column. + /// 0-based column index. + /// 0-based Offset from beginning of serialization. + /// 0-based index into the bit mask for the null bit. + /// For bool fields, 0-based index into the bit mask for the bool value. + /// For variable length types the length, otherwise 0. + /// + /// For types with generic parameters (e.g. , the type + /// parameters. + /// + internal LayoutColumn( + string path, + LayoutType type, + TypeArgumentList typeArgs, + StorageKind storage, + LayoutColumn parent, + int index, + int offset, + LayoutBit nullBit, + LayoutBit boolBit, + int length = 0) + { + this.path = Utf8String.TranscodeUtf16(path); + this.fullPath = Utf8String.TranscodeUtf16(LayoutColumn.GetFullPath(parent, path)); + this.type = type; + this.typeArgs = typeArgs; + this.typeArg = new TypeArgument(type, typeArgs); + this.storage = storage; + this.parent = parent; + this.index = index; + this.offset = offset; + this.nullBit = nullBit; + this.boolBit = boolBit; + this.size = this.typeArg.Type.IsFixed ? type.Size : length; + } + + /// The relative path of the field within its parent scope. + /// + /// Paths are expressed in dotted notation: e.g. a relative of 'b.c' + /// within the scope 'a' yields a of 'a.b.c'. + /// + public Utf8String Path => this.path; + + /// The full logical path of the field within the row. + /// + /// Paths are expressed in dotted notation: e.g. a relative of 'b.c' + /// within the scope 'a' yields a of 'a.b.c'. + /// + public Utf8String FullPath => this.fullPath; + + /// The physical layout type of the field. + public LayoutType Type => this.type; + + /// The storage kind of the field. + public StorageKind Storage => this.storage; + + /// The layout of the parent scope, if a nested column, otherwise null. + public LayoutColumn Parent => this.parent; + + /// The full logical type. + public TypeArgument TypeArg => this.typeArg; + + /// For types with generic parameters (e.g. , the type parameters. + public TypeArgumentList TypeArgs => this.typeArgs; + + /// + /// 0-based index of the column within the structure. Also indicates which presence bit + /// controls this column. + /// + public int Index => this.index; + + /// + /// If equals then the byte offset to + /// the field location. + /// + /// If equals then the 0-based index of the + /// field from the beginning of the variable length segment. + /// + /// For all other values of , is ignored. + /// + public int Offset + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.offset; + } + + /// For nullable fields, the the bit in the layout bitmask for the null bit. + public LayoutBit NullBit + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.nullBit; + } + + /// For bool fields, 0-based index into the bit mask for the bool value. + public LayoutBit BoolBit + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.boolBit; + } + + /// + /// If equals then the fixed number of + /// bytes reserved for the value. + /// + /// If equals then the maximum number of + /// bytes allowed for the value. + /// + public int Size + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.size; + } + + /// The physical layout type of the field cast to the specified type. + [DebuggerHidden] + public T TypeAs() + where T : ILayoutType + { + return this.type.TypeAs(); + } + + internal void SetIndex(int index) + { + this.index = index; + } + + internal void SetOffset(int offset) + { + this.offset = offset; + } + + /// Computes the full logical path to the column. + /// The layout of the parent scope, if a nested column, otherwise null. + /// The path to the field relative to parent scope. + /// The full logical path. + private static string GetFullPath(LayoutColumn parent, string path) + { + if (parent != null) + { + switch (LayoutCodeTraits.ClearImmutableBit(parent.type.LayoutCode)) + { + case LayoutCode.ObjectScope: + case LayoutCode.Schema: + return parent.FullPath.ToString() + "." + path; + case LayoutCode.ArrayScope: + case LayoutCode.TypedArrayScope: + case LayoutCode.TypedSetScope: + case LayoutCode.TypedMapScope: + return parent.FullPath.ToString() + "[]" + path; + default: + Contract.Fail($"Parent scope type not supported: {parent.type.LayoutCode}"); + return default; + } + } + + return path; + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutCompilationException.cs b/dotnet/src/HybridRow/Layouts/LayoutCompilationException.cs new file mode 100644 index 0000000..72fd328 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutCompilationException.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.Serialization; + + [Serializable] + [ExcludeFromCodeCoverage] + public sealed class LayoutCompilationException : Exception + { + public LayoutCompilationException() + { + } + + public LayoutCompilationException(string message) + : base(message) + { + } + + public LayoutCompilationException(string message, Exception innerException) + : base(message, innerException) + { + } + + private LayoutCompilationException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutCompiler.cs b/dotnet/src/HybridRow/Layouts/LayoutCompiler.cs new file mode 100644 index 0000000..67b6aa9 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutCompiler.cs @@ -0,0 +1,335 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + /// Converts a logical schema into a physical layout. + internal sealed class LayoutCompiler + { + /// Compiles a logical schema into a physical layout that can be used to read and write rows. + /// The namespace within which is defined. + /// The logical schema to produce a layout for. + /// The layout for the schema. + public static Layout Compile(Namespace ns, Schema schema) + { + Contract.Requires(ns != null); + Contract.Requires(schema != null); + Contract.Requires(schema.Type == TypeKind.Schema); + Contract.Requires(!string.IsNullOrWhiteSpace(schema.Name)); + Contract.Requires(ns.Schemas.Contains(schema)); + + LayoutBuilder builder = new LayoutBuilder(schema.Name, schema.SchemaId); + LayoutCompiler.AddProperties(builder, ns, LayoutCode.Schema, schema.Properties); + + return builder.Build(); + } + + private static void AddProperties(LayoutBuilder builder, Namespace ns, LayoutCode scope, List properties) + { + foreach (Property p in properties) + { + TypeArgumentList typeArgs; + LayoutType type = LayoutCompiler.LogicalToPhysicalType(ns, p.PropertyType, out typeArgs); + switch (LayoutCodeTraits.ClearImmutableBit(type.LayoutCode)) + { + case LayoutCode.ObjectScope: + { + if (!p.PropertyType.Nullable) + { + throw new LayoutCompilationException("Non-nullable sparse column are not supported."); + } + + ObjectPropertyType op = (ObjectPropertyType)p.PropertyType; + builder.AddObjectScope(p.Path, type); + LayoutCompiler.AddProperties(builder, ns, type.LayoutCode, op.Properties); + builder.EndObjectScope(); + break; + } + + case LayoutCode.ArrayScope: + case LayoutCode.TypedArrayScope: + case LayoutCode.SetScope: + case LayoutCode.TypedSetScope: + case LayoutCode.MapScope: + case LayoutCode.TypedMapScope: + case LayoutCode.TupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.Schema: + { + if (!p.PropertyType.Nullable) + { + throw new LayoutCompilationException("Non-nullable sparse column are not supported."); + } + + builder.AddTypedScope(p.Path, type, typeArgs); + break; + } + + case LayoutCode.NullableScope: + { + throw new LayoutCompilationException("Nullables cannot be explicitly declared as columns."); + } + + default: + { + PrimitivePropertyType pp = p.PropertyType as PrimitivePropertyType; + if (pp != null) + { + switch (pp.Storage) + { + case StorageKind.Fixed: + if (LayoutCodeTraits.ClearImmutableBit(scope) != LayoutCode.Schema) + { + throw new LayoutCompilationException("Cannot have fixed storage within a sparse scope."); + } + + if (type.IsNull && !pp.Nullable) + { + throw new LayoutCompilationException("Non-nullable null columns are not supported."); + } + + builder.AddFixedColumn(p.Path, type, pp.Nullable, pp.Length); + break; + case StorageKind.Variable: + if (LayoutCodeTraits.ClearImmutableBit(scope) != LayoutCode.Schema) + { + throw new LayoutCompilationException("Cannot have variable storage within a sparse scope."); + } + + if (!pp.Nullable) + { + throw new LayoutCompilationException("Non-nullable variable columns are not supported."); + } + + builder.AddVariableColumn(p.Path, type, pp.Length); + break; + case StorageKind.Sparse: + if (!pp.Nullable) + { + throw new LayoutCompilationException("Non-nullable sparse columns are not supported."); + } + + builder.AddSparseColumn(p.Path, type); + break; + default: + throw new LayoutCompilationException($"Unknown storage specification: {pp.Storage}"); + } + } + else + { + throw new LayoutCompilationException($"Unknown property type: {type.Name}"); + } + + break; + } + } + } + } + + private static LayoutType LogicalToPhysicalType(Namespace ns, PropertyType logicalType, out TypeArgumentList typeArgs) + { + typeArgs = TypeArgumentList.Empty; + bool immutable = (logicalType as ScopePropertyType)?.Immutable ?? false; + + switch (logicalType.Type) + { + case TypeKind.Null: + return LayoutType.Null; + case TypeKind.Boolean: + return LayoutType.Boolean; + case TypeKind.Int8: + return LayoutType.Int8; + case TypeKind.Int16: + return LayoutType.Int16; + case TypeKind.Int32: + return LayoutType.Int32; + case TypeKind.Int64: + return LayoutType.Int64; + case TypeKind.UInt8: + return LayoutType.UInt8; + case TypeKind.UInt16: + return LayoutType.UInt16; + case TypeKind.UInt32: + return LayoutType.UInt32; + case TypeKind.UInt64: + return LayoutType.UInt64; + case TypeKind.Float32: + return LayoutType.Float32; + case TypeKind.Float64: + return LayoutType.Float64; + case TypeKind.Float128: + return LayoutType.Float128; + case TypeKind.Decimal: + return LayoutType.Decimal; + case TypeKind.DateTime: + return LayoutType.DateTime; + case TypeKind.UnixDateTime: + return LayoutType.UnixDateTime; + case TypeKind.Guid: + return LayoutType.Guid; + case TypeKind.MongoDbObjectId: + return LayoutType.MongoDbObjectId; + case TypeKind.Utf8: + return LayoutType.Utf8; + case TypeKind.Binary: + return LayoutType.Binary; + case TypeKind.VarInt: + return LayoutType.VarInt; + case TypeKind.VarUInt: + return LayoutType.VarUInt; + + case TypeKind.Object: + return immutable ? LayoutType.ImmutableObject : LayoutType.Object; + case TypeKind.Array: + ArrayPropertyType ap = (ArrayPropertyType)logicalType; + if ((ap.Items != null) && (ap.Items.Type != TypeKind.Any)) + { + TypeArgumentList itemTypeArgs; + LayoutType itemType = LayoutCompiler.LogicalToPhysicalType(ns, ap.Items, out itemTypeArgs); + if (ap.Items.Nullable) + { + itemTypeArgs = new TypeArgumentList(new[] { new TypeArgument(itemType, itemTypeArgs) }); + itemType = itemType.Immutable ? LayoutType.ImmutableNullable : LayoutType.Nullable; + } + + typeArgs = new TypeArgumentList(new[] { new TypeArgument(itemType, itemTypeArgs) }); + return immutable ? LayoutType.ImmutableTypedArray : LayoutType.TypedArray; + } + + return immutable ? LayoutType.ImmutableArray : LayoutType.Array; + case TypeKind.Set: + SetPropertyType sp = (SetPropertyType)logicalType; + if ((sp.Items != null) && (sp.Items.Type != TypeKind.Any)) + { + TypeArgumentList itemTypeArgs; + LayoutType itemType = LayoutCompiler.LogicalToPhysicalType(ns, sp.Items, out itemTypeArgs); + if (sp.Items.Nullable) + { + itemTypeArgs = new TypeArgumentList(new[] { new TypeArgument(itemType, itemTypeArgs) }); + itemType = itemType.Immutable ? LayoutType.ImmutableNullable : LayoutType.Nullable; + } + + typeArgs = new TypeArgumentList(new[] { new TypeArgument(itemType, itemTypeArgs) }); + return immutable ? LayoutType.ImmutableTypedSet : LayoutType.TypedSet; + } + + // TODO(283638): implement sparse set. + throw new LayoutCompilationException($"Unknown property type: {logicalType.Type}"); + + case TypeKind.Map: + MapPropertyType mp = (MapPropertyType)logicalType; + if ((mp.Keys != null) && (mp.Keys.Type != TypeKind.Any) && (mp.Values != null) && (mp.Values.Type != TypeKind.Any)) + { + TypeArgumentList keyTypeArgs; + LayoutType keyType = LayoutCompiler.LogicalToPhysicalType(ns, mp.Keys, out keyTypeArgs); + if (mp.Keys.Nullable) + { + keyTypeArgs = new TypeArgumentList(new[] { new TypeArgument(keyType, keyTypeArgs) }); + keyType = keyType.Immutable ? LayoutType.ImmutableNullable : LayoutType.Nullable; + } + + TypeArgumentList valueTypeArgs; + LayoutType valueType = LayoutCompiler.LogicalToPhysicalType(ns, mp.Values, out valueTypeArgs); + if (mp.Values.Nullable) + { + valueTypeArgs = new TypeArgumentList(new[] { new TypeArgument(valueType, valueTypeArgs) }); + valueType = valueType.Immutable ? LayoutType.ImmutableNullable : LayoutType.Nullable; + } + + typeArgs = new TypeArgumentList(new[] { new TypeArgument(keyType, keyTypeArgs), new TypeArgument(valueType, valueTypeArgs) }); + return immutable ? LayoutType.ImmutableTypedMap : LayoutType.TypedMap; + } + + // TODO(283638): implement sparse map. + throw new LayoutCompilationException($"Unknown property type: {logicalType.Type}"); + + case TypeKind.Tuple: + TuplePropertyType tp = (TuplePropertyType)logicalType; + TypeArgument[] args = new TypeArgument[tp.Items.Count]; + for (int i = 0; i < tp.Items.Count; i++) + { + TypeArgumentList itemTypeArgs; + LayoutType itemType = LayoutCompiler.LogicalToPhysicalType(ns, tp.Items[i], out itemTypeArgs); + if (tp.Items[i].Nullable) + { + itemTypeArgs = new TypeArgumentList(new[] { new TypeArgument(itemType, itemTypeArgs) }); + itemType = itemType.Immutable ? LayoutType.ImmutableNullable : LayoutType.Nullable; + } + + args[i] = new TypeArgument(itemType, itemTypeArgs); + } + + typeArgs = new TypeArgumentList(args); + return immutable ? LayoutType.ImmutableTypedTuple : LayoutType.TypedTuple; + + case TypeKind.Tagged: + TaggedPropertyType tg = (TaggedPropertyType)logicalType; + if ((tg.Items.Count < TaggedPropertyType.MinTaggedArguments) || (tg.Items.Count > TaggedPropertyType.MaxTaggedArguments)) + { + throw new LayoutCompilationException( + $"Invalid number of arguments in Tagged: {TaggedPropertyType.MinTaggedArguments} <= {tg.Items.Count} <= {TaggedPropertyType.MaxTaggedArguments}"); + } + + TypeArgument[] tgArgs = new TypeArgument[tg.Items.Count + 1]; + tgArgs[0] = new TypeArgument(LayoutType.UInt8, TypeArgumentList.Empty); + for (int i = 0; i < tg.Items.Count; i++) + { + TypeArgumentList itemTypeArgs; + LayoutType itemType = LayoutCompiler.LogicalToPhysicalType(ns, tg.Items[i], out itemTypeArgs); + if (tg.Items[i].Nullable) + { + itemTypeArgs = new TypeArgumentList(new[] { new TypeArgument(itemType, itemTypeArgs) }); + itemType = itemType.Immutable ? LayoutType.ImmutableNullable : LayoutType.Nullable; + } + + tgArgs[i + 1] = new TypeArgument(itemType, itemTypeArgs); + } + + typeArgs = new TypeArgumentList(tgArgs); + switch (tg.Items.Count) + { + case 1: + return immutable ? LayoutType.ImmutableTagged : LayoutType.Tagged; + case 2: + return immutable ? LayoutType.ImmutableTagged2 : LayoutType.Tagged2; + default: + throw new LayoutCompilationException("Unexpected tagged arity"); + } + + case TypeKind.Schema: + UdtPropertyType up = (UdtPropertyType)logicalType; + Schema udtSchema; + if (up.SchemaId == SchemaId.Invalid) + { + udtSchema = ns.Schemas.Find(s => s.Name == up.Name); + } + else + { + udtSchema = ns.Schemas.Find(s => s.SchemaId == up.SchemaId); + if (udtSchema.Name != up.Name) + { + throw new LayoutCompilationException($"Ambiguous schema reference: '{up.Name}:{up.SchemaId}'"); + } + } + + if (udtSchema == null) + { + throw new LayoutCompilationException($"Cannot resolve schema reference '{up.Name}:{up.SchemaId}'"); + } + + typeArgs = new TypeArgumentList(udtSchema.SchemaId); + return immutable ? LayoutType.ImmutableUDT : LayoutType.UDT; + + default: + throw new LayoutCompilationException($"Unknown property type: {logicalType.Type}"); + } + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutResolver.cs b/dotnet/src/HybridRow/Layouts/LayoutResolver.cs new file mode 100644 index 0000000..73c2e6f --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutResolver.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + public abstract class LayoutResolver + { + public abstract Layout Resolve(SchemaId schemaId); + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutResolverNamespace.cs b/dotnet/src/HybridRow/Layouts/LayoutResolverNamespace.cs new file mode 100644 index 0000000..6d375a1 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutResolverNamespace.cs @@ -0,0 +1,64 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System.Collections.Concurrent; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + /// + /// An implementation of which dynamically compiles schema from + /// a . + /// + /// + /// This resolver assumes that within the have + /// their properly populated. The resolver caches compiled schema. + /// + /// All members of this class are multi-thread safe. + public sealed class LayoutResolverNamespace : LayoutResolver + { + private readonly ConcurrentDictionary layoutCache; + private readonly LayoutResolver parent; + private readonly Namespace schemaNamespace; + + public LayoutResolverNamespace(Namespace schemaNamespace, LayoutResolver parent = default) + { + this.schemaNamespace = schemaNamespace; + this.parent = parent; + this.layoutCache = new ConcurrentDictionary(); + } + + public Namespace Namespace => this.schemaNamespace; + + public override Layout Resolve(SchemaId schemaId) + { + if (this.layoutCache.TryGetValue(schemaId.Id, out Layout layout)) + { + return layout; + } + + foreach (Schema s in this.schemaNamespace.Schemas) + { + if (s.SchemaId == schemaId) + { + layout = s.Compile(this.schemaNamespace); + layout = this.layoutCache.GetOrAdd(schemaId.Id, layout); + return layout; + } + } + + layout = this.parent?.Resolve(schemaId); + if (layout != null) + { + bool succeeded = this.layoutCache.TryAdd(schemaId.Id, layout); + Contract.Assert(succeeded); + return layout; + } + + Contract.Fail($"Failed to resolve schema {schemaId}"); + return null; + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutResolverSimple.cs b/dotnet/src/HybridRow/Layouts/LayoutResolverSimple.cs new file mode 100644 index 0000000..2d4245c --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutResolverSimple.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + + public sealed class LayoutResolverSimple : LayoutResolver + { + private readonly Func resolver; + + public LayoutResolverSimple(Func resolver) + { + this.resolver = resolver; + } + + public override Layout Resolve(SchemaId schemaId) + { + return this.resolver(schemaId); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/LayoutType.cs b/dotnet/src/HybridRow/Layouts/LayoutType.cs new file mode 100644 index 0000000..f3716a6 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/LayoutType.cs @@ -0,0 +1,3337 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1402 // FileMayOnlyContainASingleType +#pragma warning disable SA1201 // Elements should appear in the correct order +#pragma warning disable SA1401 // Fields should be private +#pragma warning disable CA1040 // Avoid empty interfaces +#pragma warning disable CA1051 // Do not declare visible instance fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Buffers; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + + /// The abstract base class for typed hybrid row field descriptors. + /// is immutable. + [DebuggerDisplay("{" + nameof(LayoutType.Name) + "}")] + public abstract class LayoutType : ILayoutType + { + /// The number of bits in a single byte on the current architecture. + internal const int BitsPerByte = 8; + + private static readonly LayoutType[] CodeIndex = new LayoutType[(int)LayoutCode.EndScope + 1]; + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutInt8 Int8 = new LayoutInt8(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutInt16 Int16 = new LayoutInt16(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutInt32 Int32 = new LayoutInt32(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutInt64 Int64 = new LayoutInt64(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUInt8 UInt8 = new LayoutUInt8(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUInt16 UInt16 = new LayoutUInt16(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUInt32 UInt32 = new LayoutUInt32(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUInt64 UInt64 = new LayoutUInt64(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutVarInt VarInt = new LayoutVarInt(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutVarUInt VarUInt = new LayoutVarUInt(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutFloat32 Float32 = new LayoutFloat32(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutFloat64 Float64 = new LayoutFloat64(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutFloat128 Float128 = new LayoutFloat128(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutDecimal Decimal = new LayoutDecimal(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutNull Null = new LayoutNull(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutBoolean Boolean = new LayoutBoolean(true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutBoolean BooleanFalse = new LayoutBoolean(false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutDateTime DateTime = new LayoutDateTime(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUnixDateTime UnixDateTime = new LayoutUnixDateTime(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutGuid Guid = new LayoutGuid(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutMongoDbObjectId MongoDbObjectId = new LayoutMongoDbObjectId(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUtf8 Utf8 = new LayoutUtf8(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutBinary Binary = new LayoutBinary(); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutObject Object = new LayoutObject(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutObject ImmutableObject = new LayoutObject(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutArray Array = new LayoutArray(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutArray ImmutableArray = new LayoutArray(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedArray TypedArray = new LayoutTypedArray(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedArray ImmutableTypedArray = new LayoutTypedArray(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedSet TypedSet = new LayoutTypedSet(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedSet ImmutableTypedSet = new LayoutTypedSet(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedMap TypedMap = new LayoutTypedMap(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedMap ImmutableTypedMap = new LayoutTypedMap(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTuple Tuple = new LayoutTuple(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTuple ImmutableTuple = new LayoutTuple(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedTuple TypedTuple = new LayoutTypedTuple(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTypedTuple ImmutableTypedTuple = new LayoutTypedTuple(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTagged Tagged = new LayoutTagged(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTagged ImmutableTagged = new LayoutTagged(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTagged2 Tagged2 = new LayoutTagged2(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutTagged2 ImmutableTagged2 = new LayoutTagged2(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutNullable Nullable = new LayoutNullable(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutNullable ImmutableNullable = new LayoutNullable(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUDT UDT = new LayoutUDT(immutable: false); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutUDT ImmutableUDT = new LayoutUDT(immutable: true); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + internal static readonly LayoutEndScope EndScope = new LayoutEndScope(); + + /// The physical layout code used to represent the type within the serialization. + public readonly LayoutCode LayoutCode; + + /// Initializes a new instance of the class. + internal LayoutType(LayoutCode code, bool immutable, int size) + { + this.LayoutCode = code; + this.Immutable = immutable; + this.Size = size; + LayoutType.CodeIndex[(int)code] = this; + } + + /// Human readable name of the type. + public abstract string Name { get; } + + /// True if this type is always fixed length. + public abstract bool IsFixed { get; } + + /// True if this type can be used in the variable-length segment. + public bool AllowVariable => !this.IsFixed; + + /// If true, this edit's nested fields cannot be updated individually. + /// The entire edit can still be replaced. + public readonly bool Immutable; + + /// If fixed, the fixed size of the type's serialization in bytes, otherwise undefined. + public readonly int Size; + + /// True if this type is a boolean. + public virtual bool IsBool => false; + + /// True if this type is a literal null. + public virtual bool IsNull => false; + + /// True if this type is a variable-length encoded integer type (either signed or unsigned). + public virtual bool IsVarint => false; + + /// The physical layout type of the field cast to the specified type. + [DebuggerHidden] + public T TypeAs() + where T : ILayoutType + { + return (T)(ILayoutType)this; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static LayoutType FromCode(LayoutCode code) + { + LayoutType type = LayoutType.CodeIndex[(int)code]; +#if DEBUG + if (type == null) + { + Contract.Fail($"Not Implemented: {code}"); + } +#endif + + return type; + } + + /// Helper for preparing the delete of a sparse field. + /// The row to delete from. + /// The parent edit containing the field to delete. + /// The expected type of the field. + /// Success if the delete is permitted, the error code otherwise. + internal static Result PrepareSparseDelete(ref RowBuffer b, ref RowCursor edit, LayoutCode code) + { + if (edit.scopeType.IsFixedArity) + { + return Result.TypeConstraint; + } + + if (edit.immutable) + { + return Result.InsufficientPermissions; + } + + if (edit.exists && LayoutCodeTraits.Canonicalize(edit.cellType.LayoutCode) != code) + { + return Result.TypeMismatch; + } + + return Result.Success; + } + + /// Helper for preparing the write of a sparse field. + /// The row to write to. + /// The cursor for the field to write. + /// The (optional) type constraints. + /// The write options. + /// Success if the write is permitted, the error code otherwise. + internal static Result PrepareSparseWrite(ref RowBuffer b, ref RowCursor edit, TypeArgument typeArg, UpdateOptions options) + { + if (edit.immutable || (edit.scopeType.IsUniqueScope && !edit.deferUniqueIndex)) + { + return Result.InsufficientPermissions; + } + + if (edit.scopeType.IsFixedArity && !(edit.scopeType is LayoutNullable)) + { + if ((edit.index < edit.scopeTypeArgs.Count) && !typeArg.Equals(edit.scopeTypeArgs[edit.index])) + { + return Result.TypeConstraint; + } + } + else if (edit.scopeType is LayoutTypedMap) + { + if (!((typeArg.Type is LayoutTypedTuple) && typeArg.TypeArgs.Equals(edit.scopeTypeArgs))) + { + return Result.TypeConstraint; + } + } + else if (edit.scopeType.IsTypedScope && !typeArg.Equals(edit.scopeTypeArgs[0])) + { + return Result.TypeConstraint; + } + + if ((options == UpdateOptions.InsertAt) && edit.scopeType.IsFixedArity) + { + return Result.TypeConstraint; + } + + if ((options == UpdateOptions.InsertAt) && !edit.scopeType.IsFixedArity) + { + edit.exists = false; // InsertAt never overwrites an existing item. + } + + if ((options == UpdateOptions.Update) && (!edit.exists)) + { + return Result.NotFound; + } + + if ((options == UpdateOptions.Insert) && edit.exists) + { + return Result.Exists; + } + + return Result.Success; + } + + /// Helper for preparing the read of a sparse field. + /// The row to read from. + /// The parent edit containing the field to read. + /// The expected type of the field. + /// Success if the read is permitted, the error code otherwise. + internal static Result PrepareSparseRead(ref RowBuffer b, ref RowCursor edit, LayoutCode code) + { + if (!edit.exists) + { + return Result.NotFound; + } + + if (LayoutCodeTraits.Canonicalize(edit.cellType.LayoutCode) != code) + { + return Result.TypeMismatch; + } + + return Result.Success; + } + + /// Helper for preparing the move of a sparse field into an existing restricted edit. + /// The row to read from. + /// The parent set edit into which the field should be moved. + /// The expected type of the edit moving within. + /// The expected type of the elements within the edit. + /// The field to be moved. + /// The move options. + /// If successful, a prepared insertion cursor for the destination. + /// Success if the move is permitted, the error code otherwise. + /// The source field is delete if the move prepare fails with a destination error. + internal static Result PrepareSparseMove( + ref RowBuffer b, + ref RowCursor destinationScope, + LayoutScope destinationCode, + TypeArgument elementType, + ref RowCursor srcEdit, + UpdateOptions options, + out RowCursor dstEdit) + { + Contract.Requires(destinationScope.scopeType == destinationCode); + Contract.Requires(destinationScope.index == 0, "Can only insert into a edit at the root"); + + // Prepare the delete of the source. + Result result = LayoutType.PrepareSparseDelete(ref b, ref srcEdit, elementType.Type.LayoutCode); + if (result != Result.Success) + { + dstEdit = default; + return result; + } + + if (!srcEdit.exists) + { + dstEdit = default; + return Result.NotFound; + } + + if (destinationScope.immutable) + { + b.DeleteSparse(ref srcEdit); + dstEdit = default; + return Result.InsufficientPermissions; + } + + if (!srcEdit.cellTypeArgs.Equals(elementType.TypeArgs)) + { + b.DeleteSparse(ref srcEdit); + dstEdit = default; + return Result.TypeConstraint; + } + + if (options == UpdateOptions.InsertAt) + { + b.DeleteSparse(ref srcEdit); + dstEdit = default; + return Result.TypeConstraint; + } + + // Prepare the insertion at the destination. + dstEdit = b.PrepareSparseMove(ref destinationScope, ref srcEdit); + if ((options == UpdateOptions.Update) && (!dstEdit.exists)) + { + b.DeleteSparse(ref srcEdit); + dstEdit = default; + return Result.NotFound; + } + + if ((options == UpdateOptions.Insert) && dstEdit.exists) + { + b.DeleteSparse(ref srcEdit); + dstEdit = default; + return Result.Exists; + } + + return Result.Success; + } + + internal virtual int CountTypeArgument(TypeArgumentList value) + { + return sizeof(LayoutCode); + } + + internal virtual int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + row.WriteSparseTypeCode(offset, this.LayoutCode); + return sizeof(LayoutCode); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static TypeArgument ReadTypeArgument(ref RowBuffer row, int offset, out int lenInBytes) + { + LayoutType itemCode = row.ReadSparseTypeCode(offset); + TypeArgumentList itemTypeArgs = itemCode.ReadTypeArgumentList(ref row, offset + sizeof(LayoutCode), out int argsLenInBytes); + lenInBytes = sizeof(LayoutCode) + argsLenInBytes; + return new TypeArgument(itemCode, itemTypeArgs); + } + + internal virtual TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + lenInBytes = 0; + return TypeArgumentList.Empty; + } + } + + /// Marker interface for layout types. + public interface ILayoutType + { + } + + /// + /// Describes the physical byte layout of a hybrid row field of a specific physical type + /// . + /// + /// + /// is an immutable, stateless, helper class. It provides + /// methods for manipulating hybrid row fields of a particular type, and properties that describe the + /// layout of fields of that type. + /// + /// is immutable. + /// + public abstract class LayoutType : LayoutType + { + private readonly TypeArgument typeArg; + + internal LayoutType(LayoutCode code, int size) + : base(code, false, size) + { + this.typeArg = new TypeArgument(this); + } + + internal TypeArgument TypeArg + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.typeArg; + } + + public Result HasValue(ref RowBuffer b, ref RowCursor scope, LayoutColumn col) + { + if (!b.ReadBit(scope.start, col.NullBit)) + { + return Result.NotFound; + } + + return Result.Success; + } + + public abstract Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, T value); + + public abstract Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out T value); + + public Result DeleteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + if (col.NullBit.IsInvalid) + { + // Cannot delete a non-nullable fixed column. + return Result.TypeMismatch; + } + + b.UnsetBit(scope.start, col.NullBit); + return Result.Success; + } + + /// Delete an existing value. + /// + /// If a value exists, then it is removed. The remainder of the row is resized to accomodate + /// a decrease in required space. If no value exists this operation is a no-op. + /// + public Result DeleteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + bool exists = b.ReadBit(scope.start, col.NullBit); + if (exists) + { + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + b.DeleteVariable(varOffset, this.IsVarint); + b.UnsetBit(scope.start, col.NullBit); + } + + return Result.Success; + } + + public virtual Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, T value) + { + return Result.Failure; + } + + public virtual Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out T value) + { + value = default; + return Result.Failure; + } + + /// Delete an existing value. + /// + /// If a value exists, then it is removed. The remainder of the row is resized to accomodate + /// a decrease in required space. If no value exists this operation is a no-op. + /// + public Result DeleteSparse(ref RowBuffer b, ref RowCursor edit) + { + Result result = LayoutType.PrepareSparseDelete(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + return result; + } + + b.DeleteSparse(ref edit); + return Result.Success; + } + + public abstract Result WriteSparse(ref RowBuffer b, ref RowCursor edit, T value, UpdateOptions options = UpdateOptions.Upsert); + + public abstract Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out T value); + } + + /// + /// An optional interface that indicates a can also write using a + /// . + /// + public interface ILayoutUtf8SpanWritable : ILayoutType + { + Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, Utf8Span value); + + Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, Utf8Span value); + + Result WriteSparse(ref RowBuffer b, ref RowCursor edit, Utf8Span value, UpdateOptions options = UpdateOptions.Upsert); + } + + /// + /// An optional interface that indicates a can also write using a + /// . + /// + /// The sub-element type to be written. + public interface ILayoutSpanWritable : ILayoutType + { + Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySpan value); + + Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySpan value); + + Result WriteSparse(ref RowBuffer b, ref RowCursor edit, ReadOnlySpan value, UpdateOptions options = UpdateOptions.Upsert); + } + + /// + /// An optional interface that indicates a can also write using a + /// . + /// + /// The sub-element type to be written. + public interface ILayoutSequenceWritable : ILayoutType + { + Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySequence value); + + Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySequence value); + + Result WriteSparse(ref RowBuffer b, ref RowCursor edit, ReadOnlySequence value, UpdateOptions options = UpdateOptions.Upsert); + } + + /// + /// An optional interface that indicates a can also read using a + /// . + /// + public interface ILayoutUtf8SpanReadable : ILayoutType + { + Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out Utf8Span value); + + Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out Utf8Span value); + + Result ReadSparse(ref RowBuffer b, ref RowCursor scope, out Utf8Span value); + } + + /// + /// An optional interface that indicates a can also read using a + /// . + /// + /// The sub-element type to be written. + public interface ILayoutSpanReadable : ILayoutType + { + Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ReadOnlySpan value); + + Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ReadOnlySpan value); + + Result ReadSparse(ref RowBuffer b, ref RowCursor scope, out ReadOnlySpan value); + } + + public sealed class LayoutInt8 : LayoutType + { + internal LayoutInt8() + : base(LayoutCode.Int8, size: sizeof(sbyte)) + { + } + + public override string Name => "int8"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, sbyte value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteInt8(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out sbyte value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadInt8(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse(ref RowBuffer b, ref RowCursor edit, sbyte value, UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseInt8(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out sbyte value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseInt8(ref edit); + return Result.Success; + } + } + + public sealed class LayoutInt16 : LayoutType + { + internal LayoutInt16() + : base(LayoutCode.Int16, size: sizeof(short)) + { + } + + public override string Name => "int16"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, short value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteInt16(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out short value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadInt16(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + short value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseInt16(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out short value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseInt16(ref edit); + return Result.Success; + } + } + + public sealed class LayoutInt32 : LayoutType + { + internal LayoutInt32() + : base(LayoutCode.Int32, size: sizeof(int)) + { + } + + public override string Name => "int32"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, int value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteInt32(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out int value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadInt32(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + int value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseInt32(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out int value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseInt32(ref edit); + return Result.Success; + } + } + + public sealed class LayoutInt64 : LayoutType + { + internal LayoutInt64() + : base(LayoutCode.Int64, size: sizeof(long)) + { + } + + public override string Name => "int64"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, long value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteInt64(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out long value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadInt64(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + long value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseInt64(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out long value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseInt64(ref edit); + return Result.Success; + } + } + + public sealed class LayoutUInt8 : LayoutType + { + internal LayoutUInt8() + : base(LayoutCode.UInt8, size: sizeof(byte)) + { + } + + public override string Name => "uint8"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, byte value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteUInt8(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out byte value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadUInt8(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + byte value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseUInt8(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out byte value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseUInt8(ref edit); + return Result.Success; + } + } + + public sealed class LayoutUInt16 : LayoutType + { + internal LayoutUInt16() + : base(LayoutCode.UInt16, size: sizeof(ushort)) + { + } + + public override string Name => "uint16"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ushort value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteUInt16(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ushort value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadUInt16(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + ushort value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseUInt16(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out ushort value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseUInt16(ref edit); + return Result.Success; + } + } + + public sealed class LayoutUInt32 : LayoutType + { + internal LayoutUInt32() + : base(LayoutCode.UInt32, size: sizeof(uint)) + { + } + + public override string Name => "uint32"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, uint value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteUInt32(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out uint value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadUInt32(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + uint value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseUInt32(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out uint value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseUInt32(ref edit); + return Result.Success; + } + } + + public sealed class LayoutUInt64 : LayoutType + { + internal LayoutUInt64() + : base(LayoutCode.UInt64, size: sizeof(ulong)) + { + } + + public override string Name => "uint64"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ulong value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteUInt64(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ulong value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadUInt64(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + ulong value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseUInt64(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out ulong value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseUInt64(ref edit); + return Result.Success; + } + } + + public sealed class LayoutFloat32 : LayoutType + { + internal LayoutFloat32() + : base(LayoutCode.Float32, size: sizeof(float)) + { + } + + public override string Name => "float32"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, float value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteFloat32(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out float value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadFloat32(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + float value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseFloat32(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out float value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseFloat32(ref edit); + return Result.Success; + } + } + + public sealed class LayoutFloat64 : LayoutType + { + internal LayoutFloat64() + : base(LayoutCode.Float64, size: sizeof(double)) + { + } + + public override string Name => "float64"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, double value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteFloat64(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out double value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadFloat64(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + double value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseFloat64(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out double value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseFloat64(ref edit); + return Result.Success; + } + } + + public sealed class LayoutFloat128 : LayoutType + { + internal LayoutFloat128() + : base(LayoutCode.Float128, size: HybridRow.Float128.Size) + { + } + + public override string Name => "float128"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, Float128 value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteFloat128(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out Float128 value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadFloat128(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + Float128 value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseFloat128(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out Float128 value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseFloat128(ref edit); + return Result.Success; + } + } + + public sealed class LayoutDecimal : LayoutType + { + internal LayoutDecimal() + : base(LayoutCode.Decimal, size: sizeof(decimal)) + { + } + + public override string Name => "decimal"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, decimal value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteDecimal(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out decimal value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadDecimal(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + decimal value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseDecimal(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out decimal value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseDecimal(ref edit); + return Result.Success; + } + } + + public sealed class LayoutBoolean : LayoutType + { + internal LayoutBoolean(bool value) + : base(value ? LayoutCode.Boolean : LayoutCode.BooleanFalse, size: 0) + { + } + + public override string Name => "bool"; + + public override bool IsFixed => true; + + public override bool IsBool => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, bool value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + if (value) + { + b.SetBit(scope.start, col.BoolBit); + } + else + { + b.UnsetBit(scope.start, col.BoolBit); + } + + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out bool value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadBit(scope.start, col.BoolBit); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + bool value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseBool(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out bool value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseBool(ref edit); + return Result.Success; + } + } + + public sealed class LayoutNull : LayoutType + { + internal LayoutNull() + : base(LayoutCode.Null, size: 0) + { + } + + public override string Name => "null"; + + public override bool IsFixed => true; + + public override bool IsNull => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, NullValue value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out NullValue value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + value = NullValue.Default; + if (!b.ReadBit(scope.start, col.NullBit)) + { + return Result.NotFound; + } + + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + NullValue value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseNull(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out NullValue value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseNull(ref edit); + return Result.Success; + } + } + + public sealed class LayoutDateTime : LayoutType + { + internal LayoutDateTime() + : base(LayoutCode.DateTime, size: 8) + { + } + + public override string Name => "datetime"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, DateTime value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteDateTime(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out DateTime value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadDateTime(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + DateTime value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseDateTime(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out DateTime value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseDateTime(ref edit); + return Result.Success; + } + } + + public sealed class LayoutUnixDateTime : LayoutType + { + internal LayoutUnixDateTime() + : base(LayoutCode.UnixDateTime, size: Microsoft.Azure.Cosmos.Serialization.HybridRow.UnixDateTime.Size) + { + } + + public override string Name => "unixdatetime"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, UnixDateTime value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteUnixDateTime(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out UnixDateTime value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadUnixDateTime(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + UnixDateTime value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseUnixDateTime(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out UnixDateTime value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseUnixDateTime(ref edit); + return Result.Success; + } + } + + public sealed class LayoutGuid : LayoutType + { + internal LayoutGuid() + : base(LayoutCode.Guid, size: 16) + { + } + + public override string Name => "guid"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, Guid value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteGuid(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out Guid value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadGuid(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + Guid value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseGuid(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out Guid value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseGuid(ref edit); + return Result.Success; + } + } + + public sealed class LayoutMongoDbObjectId : LayoutType + { + internal LayoutMongoDbObjectId() + : base(LayoutCode.MongoDbObjectId, size: Microsoft.Azure.Cosmos.Serialization.HybridRow.MongoDbObjectId.Size) + { + } + + // ReSharper disable once StringLiteralTypo + public override string Name => "mongodbobjectid"; + + public override bool IsFixed => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, MongoDbObjectId value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteMongoDbObjectId(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out MongoDbObjectId value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadMongoDbObjectId(scope.start + col.Offset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + MongoDbObjectId value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseMongoDbObjectId(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out MongoDbObjectId value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseMongoDbObjectId(ref edit); + return Result.Success; + } + } + + public sealed class LayoutUtf8 : LayoutType, ILayoutUtf8SpanWritable, ILayoutUtf8SpanReadable + { + internal LayoutUtf8() + : base(LayoutCode.Utf8, size: 0) + { + } + + public override string Name => "utf8"; + + public override bool IsFixed => false; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, string value) + { + Contract.Requires(value != null); + return this.WriteFixed(ref b, ref scope, col, Utf8Span.TranscodeUtf16(value)); + } + + public Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, Utf8Span value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + Contract.Requires(col.Size >= 0); + Contract.Requires(value.Length == col.Size); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteFixedString(scope.start + col.Offset, value); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out string value) + { + Result r = this.ReadFixed(ref b, ref scope, col, out Utf8Span span); + value = (r == Result.Success) ? span.ToString() : default; + return r; + } + + public Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out Utf8Span value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + Contract.Requires(col.Size >= 0); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + value = b.ReadFixedString(scope.start + col.Offset, col.Size); + return Result.Success; + } + + public override Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, string value) + { + Contract.Requires(value != null); + return this.WriteVariable(ref b, ref scope, col, Utf8Span.TranscodeUtf16(value)); + } + + public Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, Utf8Span value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + int length = value.Length; + if ((col.Size > 0) && (length > col.Size)) + { + return Result.TooBig; + } + + bool exists = b.ReadBit(scope.start, col.NullBit); + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + b.WriteVariableString(varOffset, value, exists, out int shift); + b.SetBit(scope.start, col.NullBit); + scope.metaOffset += shift; + scope.valueOffset += shift; + return Result.Success; + } + + public override Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out string value) + { + Result r = this.ReadVariable(ref b, ref scope, col, out Utf8Span span); + value = (r == Result.Success) ? span.ToString() : default; + return r; + } + + public Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out Utf8Span value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + value = b.ReadVariableString(varOffset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + string value, + UpdateOptions options = UpdateOptions.Upsert) + { + Contract.Requires(value != null); + return this.WriteSparse(ref b, ref edit, Utf8Span.TranscodeUtf16(value), options); + } + + public Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + Utf8Span value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseString(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out string value) + { + Result r = this.ReadSparse(ref b, ref edit, out Utf8Span span); + value = (r == Result.Success) ? span.ToString() : default; + return r; + } + + public Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out Utf8Span value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseString(ref edit); + return Result.Success; + } + } + + public sealed class LayoutBinary : LayoutType, ILayoutSpanWritable, ILayoutSpanReadable, ILayoutSequenceWritable + { + internal LayoutBinary() + : base(LayoutCode.Binary, size: 0) + { + } + + public override string Name => "binary"; + + public override bool IsFixed => false; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, byte[] value) + { + Contract.Requires(value != null); + return this.WriteFixed(ref b, ref scope, col, new ReadOnlySpan(value)); + } + + public Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySpan value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + Contract.Requires(col.Size >= 0); + Contract.Requires(value.Length == col.Size); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteFixedBinary(scope.start + col.Offset, value, col.Size); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySequence value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + Contract.Requires(col.Size >= 0); + Contract.Requires(value.Length == col.Size); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + b.WriteFixedBinary(scope.start + col.Offset, value, col.Size); + b.SetBit(scope.start, col.NullBit); + return Result.Success; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out byte[] value) + { + Result r = this.ReadFixed(ref b, ref scope, col, out ReadOnlySpan span); + value = (r == Result.Success) ? span.ToArray() : default; + return r; + } + + public Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ReadOnlySpan value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + Contract.Requires(col.Size >= 0); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default(byte[]); + return Result.NotFound; + } + + value = b.ReadFixedBinary(scope.start + col.Offset, col.Size); + return Result.Success; + } + + public override Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, byte[] value) + { + Contract.Requires(value != null); + return this.WriteVariable(ref b, ref scope, col, new ReadOnlySpan(value)); + } + + public Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySpan value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + int length = value.Length; + if ((col.Size > 0) && (length > col.Size)) + { + return Result.TooBig; + } + + bool exists = b.ReadBit(scope.start, col.NullBit); + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + b.WriteVariableBinary(varOffset, value, exists, out int shift); + b.SetBit(scope.start, col.NullBit); + scope.metaOffset += shift; + scope.valueOffset += shift; + return Result.Success; + } + + public Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ReadOnlySequence value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + int length = (int)value.Length; + if ((col.Size > 0) && (length > col.Size)) + { + return Result.TooBig; + } + + bool exists = b.ReadBit(scope.start, col.NullBit); + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + b.WriteVariableBinary(varOffset, value, exists, out int shift); + b.SetBit(scope.start, col.NullBit); + scope.metaOffset += shift; + scope.valueOffset += shift; + return Result.Success; + } + + public override Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out byte[] value) + { + Result r = this.ReadVariable(ref b, ref scope, col, out ReadOnlySpan span); + value = (r == Result.Success) ? span.ToArray() : default; + return r; + } + + public Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ReadOnlySpan value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default(byte[]); + return Result.NotFound; + } + + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + value = b.ReadVariableBinary(varOffset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + byte[] value, + UpdateOptions options = UpdateOptions.Upsert) + { + Contract.Requires(value != null); + return this.WriteSparse(ref b, ref edit, new ReadOnlySpan(value), options); + } + + public Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + ReadOnlySpan value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseBinary(ref edit, value, options); + return Result.Success; + } + + public Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + ReadOnlySequence value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseBinary(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out byte[] value) + { + Result r = this.ReadSparse(ref b, ref edit, out ReadOnlySpan span); + value = (r == Result.Success) ? span.ToArray() : default; + return r; + } + + public Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out ReadOnlySpan value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default(byte[]); + return result; + } + + value = b.ReadSparseBinary(ref edit); + return Result.Success; + } + } + + public sealed class LayoutVarInt : LayoutType + { + internal LayoutVarInt() + : base(LayoutCode.VarInt, size: 0) + { + } + + public override string Name => "varint"; + + public override bool IsFixed => false; + + public override bool IsVarint => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, long value) + { + Contract.Fail("Not Implemented"); + return Result.Failure; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out long value) + { + Contract.Fail("Not Implemented"); + value = default; + return Result.Failure; + } + + public override Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, long value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + bool exists = b.ReadBit(scope.start, col.NullBit); + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + b.WriteVariableInt(varOffset, value, exists, out int shift); + b.SetBit(scope.start, col.NullBit); + scope.metaOffset += shift; + scope.valueOffset += shift; + return Result.Success; + } + + public override Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out long value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + value = b.ReadVariableInt(varOffset); + return Result.Success; + } + + public override Result WriteSparse( + ref RowBuffer b, + ref RowCursor edit, + long value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseVarInt(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out long value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseVarInt(ref edit); + return Result.Success; + } + } + + public sealed class LayoutVarUInt : LayoutType + { + internal LayoutVarUInt() + : base(LayoutCode.VarUInt, size: 0) + { + } + + public override string Name => "varuint"; + + public override bool IsFixed => false; + + public override bool IsVarint => true; + + public override Result WriteFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ulong value) + { + Contract.Fail("Not Implemented"); + return Result.Failure; + } + + public override Result ReadFixed(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ulong value) + { + Contract.Fail("Not Implemented"); + value = default(long); + return Result.Failure; + } + + public override Result WriteVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, ulong value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (scope.immutable) + { + return Result.InsufficientPermissions; + } + + bool exists = b.ReadBit(scope.start, col.NullBit); + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + b.WriteVariableUInt(varOffset, value, exists, out int shift); + b.SetBit(scope.start, col.NullBit); + scope.metaOffset += shift; + scope.valueOffset += shift; + return Result.Success; + } + + public override Result ReadVariable(ref RowBuffer b, ref RowCursor scope, LayoutColumn col, out ulong value) + { + Contract.Requires(scope.scopeType is LayoutUDT); + if (!b.ReadBit(scope.start, col.NullBit)) + { + value = default; + return Result.NotFound; + } + + int varOffset = b.ComputeVariableValueOffset(scope.layout, scope.start, col.Offset); + value = b.ReadVariableUInt(varOffset); + return Result.Success; + } + + public override Result WriteSparse(ref RowBuffer b, ref RowCursor edit, ulong value, UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + return result; + } + + b.WriteSparseVarUInt(ref edit, value, options); + return Result.Success; + } + + public override Result ReadSparse(ref RowBuffer b, ref RowCursor edit, out ulong value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.ReadSparseVarUInt(ref edit); + return Result.Success; + } + } + + public abstract class LayoutScope : LayoutType + { + /// A function to write content into a . + /// The type of the context value passed by the caller. + /// The row to write to. + /// The type of the scope to write into. + /// A context value provided by the caller. + /// The result. + public delegate Result WriterFunc(ref RowBuffer b, ref RowCursor scope, TContext context); + + protected LayoutScope( + LayoutCode code, + bool immutable, + bool isSizedScope, + bool isIndexedScope, + bool isFixedArity, + bool isUniqueScope, + bool isTypedScope) + : base(code, immutable, size: 0) + { + this.IsSizedScope = isSizedScope; + this.IsIndexedScope = isIndexedScope; + this.IsFixedArity = isFixedArity; + this.IsUniqueScope = isUniqueScope; + this.IsTypedScope = isTypedScope; + } + + public sealed override bool IsFixed => false; + + public Result ReadScope(ref RowBuffer b, ref RowCursor edit, out RowCursor value) + { + Result result = LayoutType.PrepareSparseRead(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + value = default; + return result; + } + + value = b.SparseIteratorReadScope(ref edit, this.Immutable || edit.immutable || edit.scopeType.IsUniqueScope); + return Result.Success; + } + + public abstract Result WriteScope( + ref RowBuffer b, + ref RowCursor scope, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert); + + public virtual Result WriteScope( + ref RowBuffer b, + ref RowCursor scope, + TypeArgumentList typeArgs, + TContext context, + WriterFunc func, + UpdateOptions options = UpdateOptions.Upsert) + { + Result r = this.WriteScope(ref b, ref scope, typeArgs, out RowCursor childScope, options); + if (r != Result.Success) + { + return r; + } + + r = func?.Invoke(ref b, ref childScope, context) ?? Result.Success; + if (r != Result.Success) + { + this.DeleteScope(ref b, ref scope); + return r; + } + + scope.Skip(ref b, ref childScope); + return Result.Success; + } + + public Result DeleteScope(ref RowBuffer b, ref RowCursor edit) + { + Result result = LayoutType.PrepareSparseDelete(ref b, ref edit, this.LayoutCode); + if (result != Result.Success) + { + return result; + } + + b.DeleteSparse(ref edit); + return Result.Success; + } + + /// Returns true if the scope's elements cannot be updated directly. + internal readonly bool IsUniqueScope; + + /// Returns true if this is an indexed scope. + internal readonly bool IsIndexedScope; + + /// Returns true if this is a sized scope. + internal readonly bool IsSizedScope; + + /// Returns true if this is a fixed arity scope. + internal readonly bool IsFixedArity; + + /// Returns true if this is a typed scope. + internal readonly bool IsTypedScope; + + /// + /// Returns true if writing an item in the specified typed scope would elide the type code + /// because it is implied by the type arguments. + /// + /// + /// True if the type code is implied (not written), false otherwise. + internal virtual bool HasImplicitTypeCode(ref RowCursor edit) + { + return false; + } + + internal virtual void SetImplicitTypeCode(ref RowCursor edit) + { + Contract.Fail("No implicit type codes."); + } + + internal virtual void ReadSparsePath(ref RowBuffer row, ref RowCursor edit) + { + edit.pathToken = row.ReadSparsePathLen(edit.layout, edit.valueOffset, out int pathLenInBytes, out edit.pathOffset); + edit.valueOffset += pathLenInBytes; + } + } + + public sealed class LayoutEndScope : LayoutScope + { + public LayoutEndScope() + : base(LayoutCode.EndScope, false, isSizedScope: false, isIndexedScope: false, isFixedArity: false, isUniqueScope: false, isTypedScope: false) + { + } + + public override string Name => "end"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor scope, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Contract.Fail("Cannot write an EndScope directly"); + value = default; + return Result.Failure; + } + } + + public abstract class LayoutPropertyScope : LayoutScope + { + protected LayoutPropertyScope(LayoutCode code, bool immutable) + : base(code, immutable, isSizedScope: false, isIndexedScope: false, isFixedArity: false, isUniqueScope: false, isTypedScope: false) + { + } + } + + public sealed class LayoutObject : LayoutPropertyScope + { + internal LayoutObject(bool immutable) + : base(immutable ? LayoutCode.ImmutableObjectScope : LayoutCode.ObjectScope, immutable) + { + this.TypeArg = new TypeArgument(this); + } + + public override string Name => this.Immutable ? "im_object" : "object"; + + internal TypeArgument TypeArg { get; } + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteSparseObject(ref edit, this, options, out value); + return Result.Success; + } + } + + public sealed class LayoutUDT : LayoutPropertyScope + { + internal LayoutUDT(bool immutable) + : base(immutable ? LayoutCode.ImmutableSchema : LayoutCode.Schema, immutable) + { + } + + public override string Name => this.Immutable ? "im_udt" : "udt"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Layout udt = b.Resolver.Resolve(typeArgs.SchemaId); + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteSparseUDT(ref edit, this, udt, options, out value); + return Result.Success; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + return sizeof(LayoutCode) + SchemaId.Size; + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + row.WriteSparseTypeCode(offset, this.LayoutCode); + row.WriteSchemaId(offset + sizeof(LayoutCode), value.SchemaId); + return sizeof(LayoutCode) + SchemaId.Size; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + SchemaId schemaId = row.ReadSchemaId(offset); + lenInBytes = SchemaId.Size; + return new TypeArgumentList(schemaId); + } + } + + public abstract class LayoutIndexedScope : LayoutScope + { + protected LayoutIndexedScope(LayoutCode code, bool immutable, bool isSizedScope, bool isFixedArity, bool isUniqueScope, bool isTypedScope) + : base(code, immutable, isSizedScope, isIndexedScope: true, isFixedArity: isFixedArity, isUniqueScope: isUniqueScope, isTypedScope: isTypedScope) + { + } + + internal override void ReadSparsePath(ref RowBuffer row, ref RowCursor edit) + { + edit.pathToken = default; + edit.pathOffset = default; + } + } + + public sealed class LayoutArray : LayoutIndexedScope + { + internal LayoutArray(bool immutable) + : base(immutable ? LayoutCode.ImmutableArrayScope : LayoutCode.ArrayScope, immutable, isSizedScope: false, isFixedArity: false, isUniqueScope: false, isTypedScope: false) + { + this.TypeArg = new TypeArgument(this); + } + + public override string Name => this.Immutable ? "im_array" : "array"; + + internal TypeArgument TypeArg { get; } + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, this.TypeArg, options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteSparseArray(ref edit, this, options, out value); + return Result.Success; + } + } + + public sealed class LayoutTypedArray : LayoutIndexedScope + { + internal LayoutTypedArray(bool immutable) + : base(immutable ? LayoutCode.ImmutableTypedArrayScope : LayoutCode.TypedArrayScope, immutable, isSizedScope: true, isFixedArity: false, isUniqueScope: false, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_array_t" : "array_t"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteTypedArray(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index >= 0); + Contract.Assert(edit.scopeTypeArgs.Count == 1); + return !LayoutCodeTraits.AlwaysRequiresTypeCode(edit.scopeTypeArgs[0].Type.LayoutCode); + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + edit.cellType = edit.scopeTypeArgs[0].Type; + edit.cellTypeArgs = edit.scopeTypeArgs[0].TypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + Contract.Assert(value.Count == 1); + return sizeof(LayoutCode) + value[0].Type.CountTypeArgument(value[0].TypeArgs); + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + Contract.Assert(value.Count == 1); + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + lenInBytes += value[0].Type.WriteTypeArgument(ref row, offset + lenInBytes, value[0].TypeArgs); + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + return new TypeArgumentList(new[] { LayoutType.ReadTypeArgument(ref row, offset, out lenInBytes) }); + } + } + + public sealed class LayoutTuple : LayoutIndexedScope + { + internal LayoutTuple(bool immutable) + : base(immutable ? LayoutCode.ImmutableTupleScope : LayoutCode.TupleScope, immutable, isSizedScope: false, isFixedArity: true, isUniqueScope: false, isTypedScope: false) + { + } + + public override string Name => this.Immutable ? "im_tuple" : "tuple"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteSparseTuple(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + int lenInBytes = sizeof(LayoutCode); + lenInBytes += RowBuffer.Count7BitEncodedUInt((ulong)value.Count); + foreach (TypeArgument arg in value) + { + lenInBytes += arg.Type.CountTypeArgument(arg.TypeArgs); + } + + return lenInBytes; + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + lenInBytes += row.Write7BitEncodedUInt(offset + lenInBytes, (ulong)value.Count); + foreach (TypeArgument arg in value) + { + lenInBytes += arg.Type.WriteTypeArgument(ref row, offset + lenInBytes, arg.TypeArgs); + } + + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + int numTypeArgs = (int)row.Read7BitEncodedUInt(offset, out lenInBytes); + TypeArgument[] retval = new TypeArgument[numTypeArgs]; + for (int i = 0; i < numTypeArgs; i++) + { + retval[i] = LayoutType.ReadTypeArgument(ref row, offset + lenInBytes, out int itemLenInBytes); + lenInBytes += itemLenInBytes; + } + + return new TypeArgumentList(retval); + } + } + + public sealed class LayoutTypedTuple : LayoutIndexedScope + { + internal LayoutTypedTuple(bool immutable) + : base(immutable ? LayoutCode.ImmutableTypedTupleScope : LayoutCode.TypedTupleScope, immutable, isSizedScope: true, isFixedArity: true, isUniqueScope: false, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_tuple_t" : "tuple_t"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteTypedTuple(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index >= 0); + Contract.Assert(edit.scopeTypeArgs.Count > edit.index); + return !LayoutCodeTraits.AlwaysRequiresTypeCode(edit.scopeTypeArgs[edit.index].Type.LayoutCode); + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + edit.cellType = edit.scopeTypeArgs[edit.index].Type; + edit.cellTypeArgs = edit.scopeTypeArgs[edit.index].TypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + int lenInBytes = sizeof(LayoutCode); + lenInBytes += RowBuffer.Count7BitEncodedUInt((ulong)value.Count); + foreach (TypeArgument arg in value) + { + lenInBytes += arg.Type.CountTypeArgument(arg.TypeArgs); + } + + return lenInBytes; + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + lenInBytes += row.Write7BitEncodedUInt(offset + lenInBytes, (ulong)value.Count); + foreach (TypeArgument arg in value) + { + lenInBytes += arg.Type.WriteTypeArgument(ref row, offset + lenInBytes, arg.TypeArgs); + } + + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + int numTypeArgs = (int)row.Read7BitEncodedUInt(offset, out lenInBytes); + TypeArgument[] retval = new TypeArgument[numTypeArgs]; + for (int i = 0; i < numTypeArgs; i++) + { + retval[i] = LayoutType.ReadTypeArgument(ref row, offset + lenInBytes, out int itemLenInBytes); + lenInBytes += itemLenInBytes; + } + + return new TypeArgumentList(retval); + } + } + + public sealed class LayoutTagged : LayoutIndexedScope + { + internal LayoutTagged(bool immutable) + : base(immutable ? LayoutCode.ImmutableTaggedScope : LayoutCode.TaggedScope, immutable, isSizedScope: true, isFixedArity: true, isUniqueScope: false, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_tagged_t" : "tagged_t"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteTypedTuple(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index >= 0); + Contract.Assert(edit.scopeTypeArgs.Count > edit.index); + return !LayoutCodeTraits.AlwaysRequiresTypeCode(edit.scopeTypeArgs[edit.index].Type.LayoutCode); + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + edit.cellType = edit.scopeTypeArgs[edit.index].Type; + edit.cellTypeArgs = edit.scopeTypeArgs[edit.index].TypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + Contract.Assert(value.Count == 2); + return sizeof(LayoutCode) + value[1].Type.CountTypeArgument(value[1].TypeArgs); + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + Contract.Assert(value.Count == 2); + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + lenInBytes += value[1].Type.WriteTypeArgument(ref row, offset + lenInBytes, value[1].TypeArgs); + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + TypeArgument[] retval = new TypeArgument[2]; + retval[0] = new TypeArgument(LayoutType.UInt8, TypeArgumentList.Empty); + retval[1] = LayoutType.ReadTypeArgument(ref row, offset, out lenInBytes); + return new TypeArgumentList(retval); + } + } + + public sealed class LayoutTagged2 : LayoutIndexedScope + { + internal LayoutTagged2(bool immutable) + : base(immutable ? LayoutCode.ImmutableTagged2Scope : LayoutCode.Tagged2Scope, immutable, isSizedScope: true, isFixedArity: true, isUniqueScope: false, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_tagged2_t" : "tagged2_t"; + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteTypedTuple(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index >= 0); + Contract.Assert(edit.scopeTypeArgs.Count > edit.index); + return !LayoutCodeTraits.AlwaysRequiresTypeCode(edit.scopeTypeArgs[edit.index].Type.LayoutCode); + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + edit.cellType = edit.scopeTypeArgs[edit.index].Type; + edit.cellTypeArgs = edit.scopeTypeArgs[edit.index].TypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + Contract.Assert(value.Count == 3); + int lenInBytes = sizeof(LayoutCode); + for (int i = 1; i < value.Count; i++) + { + TypeArgument arg = value[i]; + lenInBytes += arg.Type.CountTypeArgument(arg.TypeArgs); + } + + return lenInBytes; + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + Contract.Assert(value.Count == 3); + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + for (int i = 1; i < value.Count; i++) + { + TypeArgument arg = value[i]; + lenInBytes += arg.Type.WriteTypeArgument(ref row, offset + lenInBytes, arg.TypeArgs); + } + + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + lenInBytes = 0; + TypeArgument[] retval = new TypeArgument[3]; + retval[0] = new TypeArgument(LayoutType.UInt8, TypeArgumentList.Empty); + for (int i = 1; i < 3; i++) + { + retval[i] = LayoutType.ReadTypeArgument(ref row, offset + lenInBytes, out int itemLenInBytes); + lenInBytes += itemLenInBytes; + } + + return new TypeArgumentList(retval); + } + } + + public sealed class LayoutNullable : LayoutIndexedScope + { + internal LayoutNullable(bool immutable) + : base(immutable ? LayoutCode.ImmutableNullableScope : LayoutCode.NullableScope, immutable, isSizedScope: true, isFixedArity: true, isUniqueScope: false, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_nullable" : "nullable"; + + public Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + bool hasValue, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteNullable(ref edit, this, typeArgs, options, hasValue, out value); + return Result.Success; + } + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + return this.WriteScope(ref b, ref edit, typeArgs, hasValue: true, out value, options); + } + + public static Result HasValue(ref RowBuffer b, ref RowCursor scope) + { + Contract.Requires(scope.scopeType is LayoutNullable); + Contract.Assert(scope.index == 1 || scope.index == 2, "Nullable scopes always point at the value"); + Contract.Assert(scope.scopeTypeArgs.Count == 1); + bool hasValue = b.ReadInt8(scope.start) != 0; + return hasValue ? Result.Success : Result.NotFound; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index >= 0); + Contract.Assert(edit.scopeTypeArgs.Count == 1); + Contract.Assert(edit.index == 1); + return !LayoutCodeTraits.AlwaysRequiresTypeCode(edit.scopeTypeArgs[0].Type.LayoutCode); + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index == 1); + edit.cellType = edit.scopeTypeArgs[0].Type; + edit.cellTypeArgs = edit.scopeTypeArgs[0].TypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + Contract.Assert(value.Count == 1); + return sizeof(LayoutCode) + value[0].Type.CountTypeArgument(value[0].TypeArgs); + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + Contract.Assert(value.Count == 1); + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + lenInBytes += value[0].Type.WriteTypeArgument(ref row, offset + lenInBytes, value[0].TypeArgs); + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + return new TypeArgumentList(new[] { LayoutType.ReadTypeArgument(ref row, offset, out lenInBytes) }); + } + } + + public abstract class LayoutUniqueScope : LayoutIndexedScope + { + protected LayoutUniqueScope(LayoutCode code, bool immutable, bool isSizedScope, bool isTypedScope) + : base(code, immutable, isSizedScope, isFixedArity: false, isUniqueScope: true, isTypedScope: isTypedScope) + { + } + + public abstract TypeArgument FieldType(ref RowCursor scope); + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor scope, + TypeArgumentList typeArgs, + TContext context, + WriterFunc func, + UpdateOptions options = UpdateOptions.Upsert) + { + Result r = this.WriteScope(ref b, ref scope, typeArgs, out RowCursor uniqueScope, options); + if (r != Result.Success) + { + return r; + } + + uniqueScope.Clone(out RowCursor childScope); + childScope.deferUniqueIndex = true; + r = func?.Invoke(ref b, ref childScope, context) ?? Result.Success; + if (r != Result.Success) + { + this.DeleteScope(ref b, ref scope); + return r; + } + + uniqueScope.count = childScope.count; + r = b.TypedCollectionUniqueIndexRebuild(ref uniqueScope); + if (r != Result.Success) + { + this.DeleteScope(ref b, ref scope); + return r; + } + + scope.Skip(ref b, ref childScope); + return Result.Success; + } + + /// Moves an existing sparse field into the unique index. + /// The row to move within. + /// The parent unique indexed edit into which the field should be moved. + /// The field to be moved. + /// The move options. + /// Success if the field is permitted within the unique index, the error code otherwise. + /// + /// The source field MUST be a field whose type arguments match the element type of the + /// destination unique index. + /// + /// The source field is delete whether the move succeeds or fails. + /// + public Result MoveField( + ref RowBuffer b, + ref RowCursor destinationScope, + ref RowCursor sourceEdit, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseMove( + ref b, + ref destinationScope, + this, + this.FieldType(ref destinationScope), + ref sourceEdit, + options, + out RowCursor dstEdit); + + if (result != Result.Success) + { + return result; + } + + // Perform the move. + b.TypedCollectionMoveField(ref dstEdit, ref sourceEdit, (RowOptions)options); + + // TODO: it would be "better" if the destinationScope were updated to point to the + // highest item seen. Then we would avoid the maximum reparse. + destinationScope.count = dstEdit.count; + return Result.Success; + } + + /// Search for a matching field within a unique index. + /// The row to search. + /// The parent unique index edit to search. + /// The parent edit from which the match pattern is read. + /// If successful, the updated edit. + /// + /// Success a matching field exists in the unique index, NotFound if no match is found, the + /// error code otherwise. + /// + /// The pattern field is delete whether the find succeeds or fails. + public Result Find(ref RowBuffer b, ref RowCursor scope, ref RowCursor patternScope, out RowCursor value) + { + Result result = LayoutType.PrepareSparseMove( + ref b, + ref scope, + this, + this.FieldType(ref scope), + ref patternScope, + UpdateOptions.Update, + out value); + + if (result != Result.Success) + { + return result; + } + + // Check if the search found the result. + b.DeleteSparse(ref patternScope); + + return Result.Success; + } + } + + public sealed class LayoutTypedSet : LayoutUniqueScope + { + internal LayoutTypedSet(bool immutable) + : base(immutable ? LayoutCode.ImmutableTypedSetScope : LayoutCode.TypedSetScope, immutable, isSizedScope: true, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_set_t" : "set_t"; + + public override TypeArgument FieldType(ref RowCursor scope) + { + return scope.scopeTypeArgs[0]; + } + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteTypedSet(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + Contract.Assert(edit.index >= 0); + Contract.Assert(edit.scopeTypeArgs.Count == 1); + return !LayoutCodeTraits.AlwaysRequiresTypeCode(edit.scopeTypeArgs[0].Type.LayoutCode); + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + edit.cellType = edit.scopeTypeArgs[0].Type; + edit.cellTypeArgs = edit.scopeTypeArgs[0].TypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + Contract.Assert(value.Count == 1); + return sizeof(LayoutCode) + value[0].Type.CountTypeArgument(value[0].TypeArgs); + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + Contract.Assert(value.Count == 1); + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + lenInBytes += value[0].Type.WriteTypeArgument(ref row, offset + lenInBytes, value[0].TypeArgs); + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + return new TypeArgumentList(new[] { LayoutType.ReadTypeArgument(ref row, offset, out lenInBytes) }); + } + } + + public sealed class LayoutTypedMap : LayoutUniqueScope + { + internal LayoutTypedMap(bool immutable) + : base(immutable ? LayoutCode.ImmutableTypedMapScope : LayoutCode.TypedMapScope, immutable, isSizedScope: true, isTypedScope: true) + { + } + + public override string Name => this.Immutable ? "im_map_t" : "map_t"; + + public override TypeArgument FieldType(ref RowCursor scope) + { + return new TypeArgument( + scope.scopeType.Immutable ? LayoutType.ImmutableTypedTuple : LayoutType.TypedTuple, + scope.scopeTypeArgs); + } + + public override Result WriteScope( + ref RowBuffer b, + ref RowCursor edit, + TypeArgumentList typeArgs, + out RowCursor value, + UpdateOptions options = UpdateOptions.Upsert) + { + Result result = LayoutType.PrepareSparseWrite(ref b, ref edit, new TypeArgument(this, typeArgs), options); + if (result != Result.Success) + { + value = default; + return result; + } + + b.WriteTypedMap(ref edit, this, typeArgs, options, out value); + return Result.Success; + } + + internal override bool HasImplicitTypeCode(ref RowCursor edit) + { + return true; + } + + internal override void SetImplicitTypeCode(ref RowCursor edit) + { + edit.cellType = edit.scopeType.Immutable ? LayoutType.ImmutableTypedTuple : LayoutType.TypedTuple; + edit.cellTypeArgs = edit.scopeTypeArgs; + } + + internal override int CountTypeArgument(TypeArgumentList value) + { + Contract.Assert(value.Count == 2); + int lenInBytes = sizeof(LayoutCode); + foreach (TypeArgument arg in value) + { + lenInBytes += arg.Type.CountTypeArgument(arg.TypeArgs); + } + + return lenInBytes; + } + + internal override int WriteTypeArgument(ref RowBuffer row, int offset, TypeArgumentList value) + { + Contract.Assert(value.Count == 2); + row.WriteSparseTypeCode(offset, this.LayoutCode); + int lenInBytes = sizeof(LayoutCode); + foreach (TypeArgument arg in value) + { + lenInBytes += arg.Type.WriteTypeArgument(ref row, offset + lenInBytes, arg.TypeArgs); + } + + return lenInBytes; + } + + internal override TypeArgumentList ReadTypeArgumentList(ref RowBuffer row, int offset, out int lenInBytes) + { + lenInBytes = 0; + TypeArgument[] retval = new TypeArgument[2]; + for (int i = 0; i < 2; i++) + { + retval[i] = LayoutType.ReadTypeArgument(ref row, offset + lenInBytes, out int itemLenInBytes); + lenInBytes += itemLenInBytes; + } + + return new TypeArgumentList(retval); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/SamplingStringComparer.cs b/dotnet/src/HybridRow/Layouts/SamplingStringComparer.cs new file mode 100644 index 0000000..d982425 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/SamplingStringComparer.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + + internal class SamplingStringComparer : IEqualityComparer + { + public static readonly SamplingStringComparer Default = new SamplingStringComparer(); + + public bool Equals(string x, string y) + { + Contract.Assert(x != null); + Contract.Assert(y != null); + + return x.Equals(y); + } + + public int GetHashCode(string obj) + { + Contract.Assert(obj != null); + + unchecked + { + uint hash1 = 5381; + uint hash2 = hash1; + const int numSamples = 4; + const int modulus = 13; + + ReadOnlySpan utf16 = obj.AsSpan(); + int max = Math.Min(utf16.Length, numSamples); + for (int i = 0; i < max; i++) + { + uint c = utf16[(i * modulus) % utf16.Length]; + if (i % 2 == 0) + { + hash1 = ((hash1 << 5) + hash1) ^ c; + } + else + { + hash2 = ((hash2 << 5) + hash2) ^ c; + } + } + + return (int)(hash1 + (hash2 * 1566083941)); + } + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/SamplingUtf8StringComparer.cs b/dotnet/src/HybridRow/Layouts/SamplingUtf8StringComparer.cs new file mode 100644 index 0000000..8ed651d --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/SamplingUtf8StringComparer.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + + internal class SamplingUtf8StringComparer : IEqualityComparer + { + public static readonly SamplingUtf8StringComparer Default = new SamplingUtf8StringComparer(); + + public bool Equals(Utf8String x, Utf8String y) + { + Contract.Assert(x != null); + Contract.Assert(y != null); + + return x.Span.Equals(y.Span); + } + + public int GetHashCode(Utf8String obj) + { + Contract.Assert(obj != null); + + unchecked + { + uint hash1 = 5381; + uint hash2 = hash1; + const int numSamples = 4; + const int modulus = 13; + + ReadOnlySpan utf8 = obj.Span.Span; + int max = Math.Min(utf8.Length, numSamples); + for (int i = 0; i < max; i++) + { + uint c = utf8[(i * modulus) % utf8.Length]; + if (i % 2 == 0) + { + hash1 = ((hash1 << 5) + hash1) ^ c; + } + else + { + hash2 = ((hash2 << 5) + hash2) ^ c; + } + } + + return (int)(hash1 + (hash2 * 1566083941)); + } + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/StringTokenizer.cs b/dotnet/src/HybridRow/Layouts/StringTokenizer.cs new file mode 100644 index 0000000..208b90d --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/StringTokenizer.cs @@ -0,0 +1,154 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Collections.Generic; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + + public sealed class StringTokenizer + { + private readonly Dictionary tokens; + private readonly Dictionary stringTokens; + private readonly List strings; + + /// Initializes a new instance of the class. + public StringTokenizer() + { + this.tokens = new Dictionary(SamplingUtf8StringComparer.Default) + { { Utf8String.Empty, new StringToken(0, Utf8String.Empty) } }; + this.stringTokens = new Dictionary(SamplingStringComparer.Default) + { { string.Empty, new StringToken(0, Utf8String.Empty) } }; + this.strings = new List { Utf8String.Empty }; + this.Count = 1; + } + + /// The number of unique tokens described by the encoding. + public int Count { get; private set; } + + /// Looks up a string's corresponding token. + /// The string to look up. + /// If successful, the string's assigned token. + /// True if successful, false otherwise. + public bool TryFindToken(UtfAnyString path, out StringToken token) + { + if (path.IsNull) + { + token = default; + return false; + } + + if (path.IsUtf8) + { + return this.tokens.TryGetValue(path.ToUtf8String(), out token); + } + + return this.stringTokens.TryGetValue(path.ToString(), out token); + } + + /// Looks up a token's corresponding string. + /// The token to look up. + /// If successful, the token's assigned string. + /// True if successful, false otherwise. + public bool TryFindString(ulong token, out Utf8String path) + { + if (token >= (ulong)this.strings.Count) + { + path = default; + return false; + } + + path = this.strings[(int)token]; + return true; + } + + /// Assign a token to the string. + /// If the string already has a token, that token is returned instead. + /// The string to assign a new token. + /// The token assigned to the string. + internal StringToken Add(Utf8String path) + { + Contract.Requires(path != null); + + if (this.tokens.TryGetValue(path, out StringToken token)) + { + return token; + } + + token = this.AllocateToken(path); + return token; + } + + /// Allocates a new token and assigns the string to it. + /// The string that needs a new token. + /// The new allocated token. + private StringToken AllocateToken(Utf8String path) + { + ulong id = (ulong)this.Count++; + StringToken token = new StringToken(id, path); + this.tokens.Add(path, token); + this.stringTokens.Add(path.ToString(), token); + this.strings.Add(path); + Contract.Assert((ulong)this.strings.Count - 1 == id); + return token; + } + } + + public readonly struct StringToken + { +#pragma warning disable CA1051 // Do not declare visible instance fields + public readonly ulong Id; + public readonly byte[] Varint; + public readonly Utf8String Path; +#pragma warning restore CA1051 // Do not declare visible instance fields + + public StringToken(ulong id, Utf8String path) + { + this.Id = id; + this.Varint = new byte[StringToken.Count7BitEncodedUInt(id)]; + StringToken.Write7BitEncodedUInt(this.Varint.AsSpan(), id); + this.Path = path; + } + + public bool IsNull + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.Varint == null; + } + + private static int Write7BitEncodedUInt(Span buffer, ulong value) + { + // Write out an unsigned long 7 bits at a time. The high bit of the byte, + // when set, indicates there are more bytes. + int i = 0; + while (value >= 0x80) + { + buffer[i] = unchecked((byte)(value | 0x80)); + i++; + value >>= 7; + } + + buffer[i] = (byte)value; + i++; + return i; + } + + private static int Count7BitEncodedUInt(ulong value) + { + // Count the number of bytes needed to write out an int 7 bits at a time. + int i = 0; + while (value >= 0x80) + { + i++; + value >>= 7; + } + + i++; + return i; + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/SystemSchema.cs b/dotnet/src/HybridRow/Layouts/SystemSchema.cs new file mode 100644 index 0000000..61d73a5 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/SystemSchema.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Reflection; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public static class SystemSchema + { + /// + /// SchemaId of the empty schema. This schema has no defined cells but can accomodate + /// unschematized sparse content. + /// + public static readonly SchemaId EmptySchemaId = new SchemaId(2147473650); + + /// SchemaId of HybridRow RecordIO Segments. + public static readonly SchemaId SegmentSchemaId = new SchemaId(2147473648); + + /// SchemaId of HybridRow RecordIO Record Headers. + public static readonly SchemaId RecordSchemaId = new SchemaId(2147473649); + + [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes", Justification = "Type is immutable.")] + public static readonly LayoutResolver LayoutResolver = SystemSchema.LoadSchema(); + + private static LayoutResolver LoadSchema() + { + string json = SystemSchema.GetEmbeddedResource(@"SystemSchemas\SystemSchema.json"); + Namespace ns = Namespace.Parse(json); + LayoutResolverNamespace resolver = new LayoutResolverNamespace(ns); + return resolver; + } + + private static string GetEmbeddedResource(string resourceName) + { + Assembly assembly = Assembly.GetAssembly(typeof(RecordIOFormatter)); + resourceName = SystemSchema.FormatResourceName(assembly, resourceName); + using (Stream resourceStream = assembly.GetManifestResourceStream(resourceName)) + { + if (resourceStream == null) + { + return null; + } + + using (StreamReader reader = new StreamReader(resourceStream)) + { + return reader.ReadToEnd(); + } + } + } + + private static string FormatResourceName(Assembly assembly, string resourceName) + { + return assembly.GetName().Name + "." + resourceName.Replace(" ", "_").Replace("\\", ".").Replace("/", "."); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/TypeArgument.cs b/dotnet/src/HybridRow/Layouts/TypeArgument.cs new file mode 100644 index 0000000..5ad9af4 --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/TypeArgument.cs @@ -0,0 +1,100 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Diagnostics; + using Microsoft.Azure.Cosmos.Core; + + [DebuggerDisplay("{this.type == null ? null : ToString()}")] + public readonly struct TypeArgument : IEquatable + { + private readonly LayoutType type; + private readonly TypeArgumentList typeArgs; + + /// Initializes a new instance of the struct. + /// The type of the constraint. + public TypeArgument(LayoutType type) + { + Contract.Requires(type != null); + + this.type = type; + this.typeArgs = TypeArgumentList.Empty; + } + + /// Initializes a new instance of the struct. + /// The type of the constraint. + /// For generic types the type parameters. + public TypeArgument(LayoutType type, TypeArgumentList typeArgs) + { + Contract.Requires(type != null); + + this.type = type; + this.typeArgs = typeArgs; + } + + /// The physical layout type. + public LayoutType Type => this.type; + + /// If the type argument is itself generic, then its type arguments. + public TypeArgumentList TypeArgs => this.typeArgs; + + public static bool operator ==(TypeArgument left, TypeArgument right) + { + return left.Equals(right); + } + + public static bool operator !=(TypeArgument left, TypeArgument right) + { + return !left.Equals(right); + } + + /// The physical layout type of the field cast to the specified type. + [DebuggerHidden] + public T TypeAs() + where T : ILayoutType + { + return this.type.TypeAs(); + } + + public override string ToString() + { + if (this.type == null) + { + return string.Empty; + } + + return this.type.Name + this.typeArgs.ToString(); + } + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (obj is TypeArgument ota) + { + return this.Equals(ota); + } + + return false; + } + + public override int GetHashCode() + { + unchecked + { + return (this.type.GetHashCode() * 397) ^ this.typeArgs.GetHashCode(); + } + } + + public bool Equals(TypeArgument other) + { + return this.type.Equals(other.type) && this.typeArgs.Equals(other.typeArgs); + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/TypeArgumentList.cs b/dotnet/src/HybridRow/Layouts/TypeArgumentList.cs new file mode 100644 index 0000000..d46991a --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/TypeArgumentList.cs @@ -0,0 +1,154 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1034 // Nested types should not be visible + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + using System; + using System.Diagnostics; + using System.Linq; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Core; + + [DebuggerDisplay("{this.args == null ? null : ToString()}")] + public readonly struct TypeArgumentList : IEquatable + { + public static readonly TypeArgumentList Empty = new TypeArgumentList(Array.Empty()); + + private readonly TypeArgument[] args; + + /// For UDT fields, the schema id of the nested layout. + private readonly SchemaId schemaId; + + public TypeArgumentList(TypeArgument[] args) + { + Contract.Requires(args != null); + + this.args = args; + this.schemaId = SchemaId.Invalid; + } + + /// Initializes a new instance of the struct. + /// For UDT fields, the schema id of the nested layout. + public TypeArgumentList(SchemaId schemaId) + { + this.args = Array.Empty(); + this.schemaId = schemaId; + } + + public int Count => this.args.Length; + + /// For UDT fields, the schema id of the nested layout. + public SchemaId SchemaId => this.schemaId; + + public TypeArgument this[int i] => this.args[i]; + + public static bool operator ==(TypeArgumentList left, TypeArgumentList right) + { + return left.Equals(right); + } + + public static bool operator !=(TypeArgumentList left, TypeArgumentList right) + { + return !left.Equals(right); + } + + /// Gets an enumerator for this span. + public Enumerator GetEnumerator() + { + return new Enumerator(this.args); + } + + public override string ToString() + { + if (this.schemaId != SchemaId.Invalid) + { + return $"<{this.schemaId.ToString()}>"; + } + + if (this.args == null || this.args?.Length == 0) + { + return string.Empty; + } + + return $"<{string.Join(", ", this.args)}>"; + } + + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + if (obj is TypeArgumentList ota) + { + return this.Equals(ota); + } + + return false; + } + + public override int GetHashCode() + { + unchecked + { + int hash = 19; + hash = (hash * 397) ^ this.schemaId.GetHashCode(); + foreach (TypeArgument a in this.args) + { + hash = (hash * 397) ^ a.GetHashCode(); + } + + return hash; + } + } + + public bool Equals(TypeArgumentList other) + { + return (this.schemaId == other.schemaId) && this.args.SequenceEqual(other.args); + } + + /// Enumerates the elements of a . + public struct Enumerator + { + /// The list being enumerated. + private readonly TypeArgument[] list; + + /// The next index to yield. + private int index; + + /// Initializes a new instance of the struct. + /// The list to enumerate. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Enumerator(TypeArgument[] list) + { + this.list = list; + this.index = -1; + } + + /// Advances the enumerator to the next element of the span. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool MoveNext() + { + int i = this.index + 1; + if (i < this.list.Length) + { + this.index = i; + return true; + } + + return false; + } + + /// Gets the element at the current position of the enumerator. + public ref readonly TypeArgument Current + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => ref this.list[this.index]; + } + } + } +} diff --git a/dotnet/src/HybridRow/Layouts/UpdateOptions.cs b/dotnet/src/HybridRow/Layouts/UpdateOptions.cs new file mode 100644 index 0000000..b54cb1a --- /dev/null +++ b/dotnet/src/HybridRow/Layouts/UpdateOptions.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts +{ + /// Describes the desired behavior when writing a . + public enum UpdateOptions + { + None = 0, + + /// Overwrite an existing value. + /// + /// An existing value is assumed to exist at the offset provided. The existing value is + /// replaced inline. The remainder of the row is resized to accomodate either an increase or decrease + /// in required space. + /// + Update = 1, + + /// Insert a new value. + /// + /// An existing value is assumed NOT to exist at the offset provided. The new value is + /// inserted immediately at the offset. The remainder of the row is resized to accomodate either an + /// increase or decrease in required space. + /// + Insert = 2, + + /// Update an existing value or insert a new value, if no value exists. + /// + /// If a value exists, then this operation becomes , otherwise it becomes + /// . + /// + Upsert = 3, + + /// Insert a new value moving existing values to the right. + /// + /// Within an array scope, inserts a new value immediately at the index moving all subsequent + /// items to the right. In any other scope behaves the same as . + /// + InsertAt = 4, + } +} diff --git a/dotnet/src/HybridRow/MemorySpanResizer.cs b/dotnet/src/HybridRow/MemorySpanResizer.cs new file mode 100644 index 0000000..17b6a57 --- /dev/null +++ b/dotnet/src/HybridRow/MemorySpanResizer.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + using Microsoft.Azure.Cosmos.Core; + + public sealed class MemorySpanResizer : ISpanResizer + { + private Memory memory; + + public MemorySpanResizer(int initialCapacity = 0) + { + Contract.Requires(initialCapacity >= 0); + + this.memory = initialCapacity == 0 ? default : new Memory(new T[initialCapacity]); + } + + public Memory Memory => this.memory; + + /// + public Span Resize(int minimumLength, Span buffer = default) + { + if (this.memory.Length < minimumLength) + { + this.memory = new Memory(new T[Math.Max(minimumLength, buffer.Length)]); + } + + Span next = this.memory.Span; + if (!buffer.IsEmpty && next.Slice(0, buffer.Length) != buffer) + { + buffer.CopyTo(next); + } + + return next; + } + } +} diff --git a/dotnet/src/HybridRow/Microsoft.Azure.Cosmos.Serialization.HybridRow.csproj b/dotnet/src/HybridRow/Microsoft.Azure.Cosmos.Serialization.HybridRow.csproj new file mode 100644 index 0000000..2ef68aa --- /dev/null +++ b/dotnet/src/HybridRow/Microsoft.Azure.Cosmos.Serialization.HybridRow.csproj @@ -0,0 +1,31 @@ + + + + true + true + {490D42EE-1FEF-47CC-97E4-782A353B4D58} + Library + Microsoft.Azure.Cosmos.Serialization.HybridRow + Microsoft.Azure.Cosmos.Serialization.HybridRow + netstandard2.0 + AnyCPU + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/HybridRow/MongoDBObjectId.cs b/dotnet/src/HybridRow/MongoDBObjectId.cs new file mode 100644 index 0000000..4daeb3c --- /dev/null +++ b/dotnet/src/HybridRow/MongoDBObjectId.cs @@ -0,0 +1,152 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + using System.Runtime.InteropServices; + using Microsoft.Azure.Cosmos.Core; + + /// A 12-byte MongoDB Object Identifier (in big-endian byte order). + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public unsafe struct MongoDbObjectId : IEquatable + { + /// The size (in bytes) of a MongoObjectId. + public const int Size = 12; + + /// The object id bytes inlined. + private fixed byte data[MongoDbObjectId.Size]; + + /// Initializes a new instance of the struct. + /// the high-order 32-bits. + /// the low-order 64-bits. + public MongoDbObjectId(uint high, ulong low) + { + Contract.Assert(BitConverter.IsLittleEndian); + + fixed (byte* p = this.data) + { + *(uint*)&p[0] = MongoDbObjectId.SwapByteOrder(high); + *(ulong*)&p[4] = MongoDbObjectId.SwapByteOrder(low); + } + } + + /// Initializes a new instance of the struct. + /// the bytes of the object id in big-endian order. + public MongoDbObjectId(ReadOnlySpan src) + { + Contract.Requires(src.Length == MongoDbObjectId.Size); + + fixed (byte* p = this.data) + fixed (byte* q = src) + { + *(ulong*)&p[0] = *(ulong*)&q[0]; + *(uint*)&p[8] = *(uint*)&q[8]; + } + } + + /// Operator == overload. + public static bool operator ==(MongoDbObjectId left, MongoDbObjectId right) + { + return left.Equals(right); + } + + /// Operator != overload. + public static bool operator !=(MongoDbObjectId left, MongoDbObjectId right) + { + return !left.Equals(right); + } + + /// Returns true if this is the same value as . + /// The value to compare against. + /// True if the two values are the same. + public bool Equals(MongoDbObjectId other) + { + fixed (byte* p = this.data) + { + byte* q = other.data; + if (*(ulong*)&q[0] != *(ulong*)&p[0] || *(uint*)&q[8] != *(uint*)&p[8]) + { + return false; + } + } + + return true; + } + + /// overload. + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + return obj is MongoDbObjectId && this.Equals((MongoDbObjectId)obj); + } + + /// overload. + public override int GetHashCode() + { + unchecked + { + int hashCode = 0; + fixed (byte* p = this.data) + { + hashCode = (hashCode * 397) ^ *(int*)&p[0]; + hashCode = (hashCode * 397) ^ *(int*)&p[4]; + hashCode = (hashCode * 397) ^ *(int*)&p[8]; + } + + return hashCode; + } + } + + /// Returns the bytes of the object id as a byte array in big-endian order. + public byte[] ToByteArray() + { + byte[] bytes = new byte[MongoDbObjectId.Size]; + this.CopyTo(bytes); + return bytes; + } + + /// Copies the bytes of the object id to the provided buffer. + /// A buffer to receive the bytes in big-endian order. + /// + /// Required: The buffer must be able to accomodate the full object id (see + /// ) at the offset indicated. + /// + public void CopyTo(Span dest) + { + Contract.Requires(dest.Length == MongoDbObjectId.Size); + + fixed (byte* p = this.data) + { + Span source = new Span(p, MongoDbObjectId.Size); + source.CopyTo(dest); + } + } + + private static uint SwapByteOrder(uint value) + { + return ((value & 0x000000FFU) << 24) | + ((value & 0x0000FF00U) << 8) | + ((value & 0x00FF0000U) >> 8) | + ((value & 0xFF000000U) >> 24); + } + + // reverse byte order (64-bit) + private static ulong SwapByteOrder(ulong value) + { + return ((value & 0x00000000000000FFUL) << 56) | + ((value & 0x000000000000FF00UL) << 40) | + ((value & 0x0000000000FF0000UL) << 24) | + ((value & 0x00000000FF000000UL) << 8) | + ((value & 0x000000FF00000000UL) >> 8) | + ((value & 0x0000FF0000000000UL) >> 24) | + ((value & 0x00FF000000000000UL) >> 40) | + ((value & 0xFF00000000000000UL) >> 56); + } + } +} diff --git a/dotnet/src/HybridRow/NullValue.cs b/dotnet/src/HybridRow/NullValue.cs new file mode 100644 index 0000000..1ff40e4 --- /dev/null +++ b/dotnet/src/HybridRow/NullValue.cs @@ -0,0 +1,57 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + + /// The literal null value. + /// + /// May be stored hybrid row to indicate the literal null value. Typically this value should + /// not be used and the corresponding column should be absent from the row. + /// + public readonly struct NullValue : IEquatable + { + /// The default null literal. + /// This is the same value as default(). + public static readonly NullValue Default = default(NullValue); + + /// Operator == overload. + public static bool operator ==(NullValue left, NullValue right) + { + return left.Equals(right); + } + + /// Operator != overload. + public static bool operator !=(NullValue left, NullValue right) + { + return !left.Equals(right); + } + + /// Returns true if this is the same value as . + /// The value to compare against. + /// True if the two values are the same. + public bool Equals(NullValue other) + { + return true; + } + + /// overload. + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + return obj is NullValue && this.Equals((NullValue)obj); + } + + /// overload. + public override int GetHashCode() + { + return 42; + } + } +} diff --git a/dotnet/src/HybridRow/Properties/AssemblyInfo.cs b/dotnet/src/HybridRow/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..bcb6e23 --- /dev/null +++ b/dotnet/src/HybridRow/Properties/AssemblyInfo.cs @@ -0,0 +1,26 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRow")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("490D42EE-1FEF-47CC-97E4-782A353B4D58")] + +// Allow tests to see internals. +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Unit" + AssemblyRef.TestPublicKey)] +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Serialization.HybridRow.Tests.Perf" + AssemblyRef.TestPublicKey)] +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator" + AssemblyRef.TestPublicKey)] +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Serialization.HybridRowStress" + AssemblyRef.TestPublicKey)] diff --git a/dotnet/src/HybridRow/RecordIO/Record.cs b/dotnet/src/HybridRow/RecordIO/Record.cs new file mode 100644 index 0000000..81fa1d2 --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/Record.cs @@ -0,0 +1,20 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1051 // Do not declare visible instance fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + public struct Record + { + public int Length; + public uint Crc32; + + public Record(int length, uint crc32) + { + this.Length = length; + this.Crc32 = crc32; + } + } +} diff --git a/dotnet/src/HybridRow/RecordIO/RecordIOFormatter.cs b/dotnet/src/HybridRow/RecordIO/RecordIOFormatter.cs new file mode 100644 index 0000000..0fc6576 --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/RecordIOFormatter.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + using System; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public static class RecordIOFormatter + { + internal static readonly Layout SegmentLayout = SystemSchema.LayoutResolver.Resolve(SystemSchema.SegmentSchemaId); + internal static readonly Layout RecordLayout = SystemSchema.LayoutResolver.Resolve(SystemSchema.RecordSchemaId); + + public static Result FormatSegment(Segment segment, out RowBuffer row, ISpanResizer resizer = default) + { + resizer = resizer ?? DefaultSpanResizer.Default; + int estimatedSize = HybridRowHeader.Size + RecordIOFormatter.SegmentLayout.Size + segment.Comment?.Length ?? + 0 + segment.SDL?.Length ?? 0 + 20; + + return RecordIOFormatter.FormatObject(resizer, estimatedSize, RecordIOFormatter.SegmentLayout, segment, SegmentSerializer.Write, out row); + } + + public static Result FormatRecord(ReadOnlyMemory body, out RowBuffer row, ISpanResizer resizer = default) + { + resizer = resizer ?? DefaultSpanResizer.Default; + int estimatedSize = HybridRowHeader.Size + RecordIOFormatter.RecordLayout.Size + body.Length; + uint crc32 = Crc32.Update(0, body.Span); + Record record = new Record(body.Length, crc32); + return RecordIOFormatter.FormatObject(resizer, estimatedSize, RecordIOFormatter.RecordLayout, record, RecordSerializer.Write, out row); + } + + private static Result FormatObject( + ISpanResizer resizer, + int initialCapacity, + Layout layout, + T obj, + RowWriter.WriterFunc writer, + out RowBuffer row) + { + row = new RowBuffer(initialCapacity, resizer); + row.InitLayout(HybridRowVersion.V1, layout, SystemSchema.LayoutResolver); + Result r = RowWriter.WriteBuffer(ref row, obj, writer); + if (r != Result.Success) + { + row = default; + return r; + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow/RecordIO/RecordIOParser.cs b/dotnet/src/HybridRow/RecordIO/RecordIOParser.cs new file mode 100644 index 0000000..30e46ae --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/RecordIOParser.cs @@ -0,0 +1,229 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + using System; + using System.Runtime.InteropServices; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public struct RecordIOParser + { + private State state; + private Segment segment; + private Record record; + + /// Describes the type of Hybrid Rows produced by the parser. + public enum ProductionType + { + /// No hybrid row was produced. The parser needs more data. + None = 0, + + /// A new segment row was produced. + Segment, + + /// A record in the current segment was produced. + Record, + } + + /// The states for the internal state machine. + /// Note: numerical ordering of these states matters. + private enum State : byte + { + Start = 0, // Start: no buffers have yet been provided to the parser. + Error, // Unrecoverable parse error encountered. + NeedSegmentLength, // Parsing segment header length + NeedSegment, // Parsing segment header + NeedHeader, // Parsing HybridRow header + NeedRecord, // Parsing record header + NeedRow, // Parsing row body + } + + /// True if a valid segment has been parsed. + public bool HaveSegment => this.state >= State.NeedHeader; + + /// If a valid segment has been parsed then current active segment, otherwise undefined. + public Segment Segment + { + get + { + Contract.Requires(this.HaveSegment); + return this.segment; + } + } + + /// Processes one buffers worth of data possibly advancing the parser state. + /// The buffer to consume. + /// Indicates the type of Hybrid Row produced in . + /// If non-empty, then the body of the next record in the sequence. + /// + /// The smallest number of bytes needed to advanced the parser state further. It is + /// recommended that Process not be called again until at least this number of bytes are available. + /// + /// + /// The number of bytes consumed from the input buffer. This number may be less + /// than the total buffer size if the parser moved to a new state. + /// + /// + /// if no error + /// has occurred, otherwise a valid + /// of the last error encountered + /// during parsing. + /// + /// > + public Result Process(Memory buffer, out ProductionType type, out Memory record, out int need, out int consumed) + { + Result r = Result.Failure; + Memory b = buffer; + type = ProductionType.None; + record = default; + switch (this.state) + { + case State.Start: + { + this.state = State.NeedSegmentLength; + goto case State.NeedSegmentLength; + } + + case State.NeedSegmentLength: + { + int minimalSegmentRowSize = HybridRowHeader.Size + RecordIOFormatter.SegmentLayout.Size; + if (b.Length < minimalSegmentRowSize) + { + need = minimalSegmentRowSize; + consumed = buffer.Length - b.Length; + return Result.InsufficientBuffer; + } + + Span span = b.Span.Slice(0, minimalSegmentRowSize); + RowBuffer row = new RowBuffer(span, HybridRowVersion.V1, SystemSchema.LayoutResolver); + RowReader reader = new RowReader(ref row); + r = SegmentSerializer.Read(ref reader, out this.segment); + if (r != Result.Success) + { + break; + } + + this.state = State.NeedSegment; + goto case State.NeedSegment; + } + + case State.NeedSegment: + { + if (b.Length < this.segment.Length) + { + need = this.segment.Length; + consumed = buffer.Length - b.Length; + return Result.InsufficientBuffer; + } + + Span span = b.Span.Slice(0, this.segment.Length); + RowBuffer row = new RowBuffer(span, HybridRowVersion.V1, SystemSchema.LayoutResolver); + RowReader reader = new RowReader(ref row); + r = SegmentSerializer.Read(ref reader, out this.segment); + if (r != Result.Success) + { + break; + } + + record = b.Slice(0, span.Length); + b = b.Slice(span.Length); + need = 0; + this.state = State.NeedHeader; + consumed = buffer.Length - b.Length; + type = ProductionType.Segment; + return Result.Success; + } + + case State.NeedHeader: + { + if (b.Length < HybridRowHeader.Size) + { + need = HybridRowHeader.Size; + consumed = buffer.Length - b.Length; + return Result.InsufficientBuffer; + } + + MemoryMarshal.TryRead(b.Span, out HybridRowHeader header); + if (header.Version != HybridRowVersion.V1) + { + r = Result.InvalidRow; + break; + } + + if (header.SchemaId == SystemSchema.SegmentSchemaId) + { + goto case State.NeedSegment; + } + + if (header.SchemaId == SystemSchema.RecordSchemaId) + { + goto case State.NeedRecord; + } + + r = Result.InvalidRow; + break; + } + + case State.NeedRecord: + { + int minimalRecordRowSize = HybridRowHeader.Size + RecordIOFormatter.RecordLayout.Size; + if (b.Length < minimalRecordRowSize) + { + need = minimalRecordRowSize; + consumed = buffer.Length - b.Length; + return Result.InsufficientBuffer; + } + + Span span = b.Span.Slice(0, minimalRecordRowSize); + RowBuffer row = new RowBuffer(span, HybridRowVersion.V1, SystemSchema.LayoutResolver); + RowReader reader = new RowReader(ref row); + r = RecordSerializer.Read(ref reader, out this.record); + if (r != Result.Success) + { + break; + } + + b = b.Slice(span.Length); + this.state = State.NeedRow; + goto case State.NeedRow; + } + + case State.NeedRow: + { + if (b.Length < this.record.Length) + { + need = this.record.Length; + consumed = buffer.Length - b.Length; + return Result.InsufficientBuffer; + } + + record = b.Slice(0, this.record.Length); + + // Validate that the record has not been corrupted. + uint crc32 = Crc32.Update(0, record.Span); + if (crc32 != this.record.Crc32) + { + r = Result.InvalidRow; + break; + } + + b = b.Slice(this.record.Length); + need = 0; + this.state = State.NeedHeader; + consumed = buffer.Length - b.Length; + type = ProductionType.Record; + return Result.Success; + } + } + + this.state = State.Error; + need = 0; + consumed = buffer.Length - b.Length; + return r; + } + } +} diff --git a/dotnet/src/HybridRow/RecordIO/RecordIOStream.cs b/dotnet/src/HybridRow/RecordIO/RecordIOStream.cs new file mode 100644 index 0000000..2c6fc99 --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/RecordIOStream.cs @@ -0,0 +1,304 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + using System; + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + + public static class RecordIOStream + { + /// A function that produces RecordIO record bodies. + /// + /// Record bodies are returned as memory blocks. It is expected that each block is a + /// HybridRow, but any binary data is allowed. + /// + /// The 0-based index of the record within the segment to be produced. + /// The byte sequence of the record body's row buffer. + /// Success if the body was produced without error, the error code otherwise. + public delegate Result ProduceFunc(long index, out ReadOnlyMemory buffer); + + /// A function that produces RecordIO record bodies. + /// + /// Record bodies are returned as memory blocks. It is expected that each block is a + /// HybridRow, but any binary data is allowed. + /// + /// The 0-based index of the record within the segment to be produced. + /// + /// A tuple with: Success if the body was produced without error, the error code otherwise. + /// And, the byte sequence of the record body's row buffer. + /// + public delegate ValueTask<(Result, ReadOnlyMemory)> ProduceFuncAsync(long index); + + /// Reads an entire RecordIO stream. + /// The stream to read from. + /// + /// A (required) delegate that is called once for each record. + ///

+ /// is passed a of the byte sequence of the + /// record body's row buffer. + ///

+ ///

If returns an error then the sequence is aborted.

+ /// + /// + /// An (optional) delegate that is called once for each segment header. + ///

+ /// If is not provided then segment headers are parsed but skipped + /// over. + ///

+ ///

+ /// is passed a of the byte sequence of + /// the segment header's row buffer. + ///

+ ///

If returns an error then the sequence is aborted.

+ /// + /// Optional memory resizer. + /// Success if the stream is parsed without error, the error code otherwise. + public static async Task ReadRecordIOAsync( + this Stream stm, + Func, Result> visitRecord, + Func, Result> visitSegment = default, + MemorySpanResizer resizer = default) + { + Contract.Requires(stm != null); + Contract.Requires(visitRecord != null); + + // Create a reusable, resizable buffer if the caller didn't provide one. + resizer = resizer ?? new MemorySpanResizer(); + + RecordIOParser parser = default; + int need = 0; + Memory active = resizer.Memory; + Memory avail = default; + while (true) + { + Contract.Assert(avail.Length < active.Length); + Contract.Assert(active.Length > 0); + Contract.Assert(active.Length >= need); + + int read = await stm.ReadAsync(active.Slice(avail.Length)); + if (read == 0) + { + break; + } + + avail = active.Slice(0, avail.Length + read); + + // If there isn't enough data to move the parser forward then just read again. + if (avail.Length < need) + { + continue; + } + + // Process the available data until no more forward progress is possible. + while (avail.Length > 0) + { + // Loop around processing available data until we don't have anymore + Result r = parser.Process( + avail, + out RecordIOParser.ProductionType prodType, + out Memory record, + out need, + out int consumed); + + if ((r != Result.Success) && (r != Result.InsufficientBuffer)) + { + return r; + } + + active = active.Slice(consumed); + avail = avail.Slice(consumed); + if (avail.IsEmpty) + { + active = resizer.Memory; + } + + // If there wasn't enough data to move the parser forward then get more data. + if (r == Result.InsufficientBuffer) + { + if (need > active.Length) + { + resizer.Resize(need, avail.Span); + active = resizer.Memory; + avail = resizer.Memory.Slice(0, avail.Length); + } + + break; + } + + // Validate the Segment + if (prodType == RecordIOParser.ProductionType.Segment) + { + Contract.Assert(!record.IsEmpty); + r = visitSegment?.Invoke(record) ?? Result.Success; + if (r != Result.Success) + { + return r; + } + } + + // Consume the record. + if (prodType == RecordIOParser.ProductionType.Record) + { + Contract.Assert(!record.IsEmpty); + + r = visitRecord(record); + if (r != Result.Success) + { + return r; + } + } + } + } + + // Make sure we processed all of the available data. + Contract.Assert(avail.Length == 0); + return Result.Success; + } + + /// Writes a RecordIO segment into a stream. + /// The stream to write to. + /// The segment header to write. + /// + /// A function to produces the record bodies for the segment. + ///

+ /// The function is called until either an error is encountered or it + /// produces an empty body. An empty body terminates the segment. + ///

+ ///

If returns an error then the sequence is aborted.

+ /// + /// + /// Optional memory resizer for RecordIO metadata row buffers. + ///

+ /// Note: This should NOT be the same resizer used to process any rows as both + /// blocks of memory are used concurrently. + ///

+ /// + /// Success if the stream is written without error, the error code otherwise. + public static Task WriteRecordIOAsync( + this Stream stm, + Segment segment, + ProduceFunc produce, + MemorySpanResizer resizer = default) + { + return stm.WriteRecordIOAsync( + segment, + index => + { + Result r = produce(index, out ReadOnlyMemory buffer); + return new ValueTask<(Result, ReadOnlyMemory)>((r, buffer)); + }, + resizer); + } + + /// Writes a RecordIO segment into a stream. + /// The stream to write to. + /// The segment header to write. + /// + /// A function to produces the record bodies for the segment. + ///

+ /// The function is called until either an error is encountered or it + /// produces an empty body. An empty body terminates the segment. + ///

+ ///

If returns an error then the sequence is aborted.

+ /// + /// + /// Optional memory resizer for RecordIO metadata row buffers. + ///

+ /// Note: This should NOT be the same resizer used to process any rows as both + /// blocks of memory are used concurrently. + ///

+ /// + /// Success if the stream is written without error, the error code otherwise. + public static async Task WriteRecordIOAsync( + this Stream stm, + Segment segment, + ProduceFuncAsync produce, + MemorySpanResizer resizer = default) + { + // Create a reusable, resizable buffer if the caller didn't provide one. + resizer = resizer ?? new MemorySpanResizer(); + + // Write a RecordIO stream. + Result r = RecordIOStream.FormatSegment(segment, resizer, out Memory metadata); + if (r != Result.Success) + { + return r; + } + + await stm.WriteAsync(metadata); + + long index = 0; + while (true) + { + ReadOnlyMemory body; + (r, body) = await produce(index++); + if (r != Result.Success) + { + return r; + } + + if (body.IsEmpty) + { + break; + } + + r = RecordIOStream.FormatRow(body, resizer, out metadata); + if (r != Result.Success) + { + return r; + } + + // Metadata and Body memory blocks should not overlap since they are both in + // play at the same time. If they do this usually means that the same resizer + // was incorrectly used for both. Check the resizer parameter passed to + // WriteRecordIOAsync for metadata. + Contract.Assert(!metadata.Span.Overlaps(body.Span)); + + await stm.WriteAsync(metadata); + await stm.WriteAsync(body); + } + + return Result.Success; + } + + /// Format a segment. + /// The segment to format. + /// The resizer to use in allocating a buffer for the segment. + /// The byte sequence of the written row buffer. + /// Success if the write completes without error, the error code otherwise. + private static Result FormatSegment(Segment segment, MemorySpanResizer resizer, out Memory block) + { + Result r = RecordIOFormatter.FormatSegment(segment, out RowBuffer row, resizer); + if (r != Result.Success) + { + block = default; + return r; + } + + block = resizer.Memory.Slice(0, row.Length); + return Result.Success; + } + + /// Compute and format a record header for the given record body. + /// The body whose record header should be formatted. + /// The resizer to use in allocating a buffer for the record header. + /// The byte sequence of the written row buffer. + /// Success if the write completes without error, the error code otherwise. + private static Result FormatRow(ReadOnlyMemory body, MemorySpanResizer resizer, out Memory block) + { + Result r = RecordIOFormatter.FormatRecord(body, out RowBuffer row, resizer); + if (r != Result.Success) + { + block = default; + return r; + } + + block = resizer.Memory.Slice(0, row.Length); + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow/RecordIO/RecordSerializer.cs b/dotnet/src/HybridRow/RecordIO/RecordSerializer.cs new file mode 100644 index 0000000..037ceab --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/RecordSerializer.cs @@ -0,0 +1,62 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public static class RecordSerializer + { + public static Result Write(ref RowWriter writer, TypeArgument typeArg, Record obj) + { + Result r; + r = writer.WriteInt32("length", obj.Length); + if (r != Result.Success) + { + return r; + } + + r = writer.WriteUInt32("crc32", obj.Crc32); + if (r != Result.Success) + { + return r; + } + + return Result.Success; + } + + public static Result Read(ref RowReader reader, out Record obj) + { + obj = default; + while (reader.Read()) + { + Result r; + + // TODO: use Path tokens here. + switch (reader.Path.ToString()) + { + case "length": + r = reader.ReadInt32(out obj.Length); + if (r != Result.Success) + { + return r; + } + + break; + case "crc32": + r = reader.ReadUInt32(out obj.Crc32); + if (r != Result.Success) + { + return r; + } + + break; + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow/RecordIO/Segment.cs b/dotnet/src/HybridRow/RecordIO/Segment.cs new file mode 100644 index 0000000..7bc5982 --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/Segment.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1051 // Do not declare visible instance fields + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + public struct Segment + { + public int Length; + public string Comment; + public string SDL; + + public Segment(string comment, string sdl) + { + this.Length = 0; + this.Comment = comment; + this.SDL = sdl; + } + } +} diff --git a/dotnet/src/HybridRow/RecordIO/SegmentSerializer.cs b/dotnet/src/HybridRow/RecordIO/SegmentSerializer.cs new file mode 100644 index 0000000..c03f646 --- /dev/null +++ b/dotnet/src/HybridRow/RecordIO/SegmentSerializer.cs @@ -0,0 +1,103 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO +{ + using System; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public static class SegmentSerializer + { + public static Result Write(ref RowWriter writer, TypeArgument typeArg, Segment obj) + { + Result r; + if (obj.Comment != null) + { + r = writer.WriteString("comment", obj.Comment); + if (r != Result.Success) + { + return r; + } + } + + if (obj.SDL != null) + { + r = writer.WriteString("sdl", obj.SDL); + if (r != Result.Success) + { + return r; + } + } + + // Defer writing the length until all other fields of the segment header are written. + // The length is then computed based on the current size of the underlying RowBuffer. + // Because the length field is itself fixed, writing the length can never change the length. + int length = writer.Length; + r = writer.WriteInt32("length", length); + if (r != Result.Success) + { + return r; + } + + Contract.Assert(length == writer.Length); + return Result.Success; + } + + public static Result Read(Span span, LayoutResolver resolver, out Segment obj) + { + RowBuffer row = new RowBuffer(span, HybridRowVersion.V1, resolver); + RowReader reader = new RowReader(ref row); + return SegmentSerializer.Read(ref reader, out obj); + } + + public static Result Read(ref RowReader reader, out Segment obj) + { + obj = default; + while (reader.Read()) + { + Result r; + + // TODO: use Path tokens here. + switch (reader.Path.ToString()) + { + case "length": + r = reader.ReadInt32(out obj.Length); + if (r != Result.Success) + { + return r; + } + + // If the RowBuffer isn't big enough to contain the rest of the header, then just + // return the length. + if (reader.Length < obj.Length) + { + return Result.Success; + } + + break; + case "comment": + r = reader.ReadString(out obj.Comment); + if (r != Result.Success) + { + return r; + } + + break; + case "sdl": + r = reader.ReadString(out obj.SDL); + if (r != Result.Success) + { + return r; + } + + break; + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRow/Result.cs b/dotnet/src/HybridRow/Result.cs new file mode 100644 index 0000000..63ab626 --- /dev/null +++ b/dotnet/src/HybridRow/Result.cs @@ -0,0 +1,45 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + public enum Result + { + Success = 0, + Failure = 1, + NotFound = 2, + Exists = 3, + TooBig = 4, + + /// + /// The type of an existing field does not match the expected type for this operation. + /// + TypeMismatch = 5, + + /// + /// An attempt to write in a read-only scope. + /// + InsufficientPermissions = 6, + + /// + /// An attempt to write a field that did not match its (optional) type constraints. + /// + TypeConstraint = 7, + + /// + /// The byte sequence could not be parsed as a valid row. + /// + InvalidRow = 8, + + /// + /// The byte sequence was too short for the requested action. + /// + InsufficientBuffer = 9, + + /// + /// The operation was cancelled. + /// + Canceled = 10, + } +} diff --git a/dotnet/src/HybridRow/RowBuffer.cs b/dotnet/src/HybridRow/RowBuffer.cs new file mode 100644 index 0000000..5f9ee02 --- /dev/null +++ b/dotnet/src/HybridRow/RowBuffer.cs @@ -0,0 +1,2957 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + using System.Buffers; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Runtime.CompilerServices; + using System.Runtime.InteropServices; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public ref struct RowBuffer + { + /// Resizer for growing the memory buffer. + private readonly ISpanResizer resizer; + + /// A sequence of bytes managed by this . + /// + /// A Hybrid Row begins in the 0-th byte of the . Remaining byte + /// sequence is defined by the Hybrid Row grammar. + /// + private Span buffer; + + /// The resolver for UDTs. + private LayoutResolver resolver; + + /// The length of row in bytes. + private int length; + + /// Initializes a new instance of the struct. + /// Initial buffer capacity. + /// Optional memory resizer. + public RowBuffer(int capacity, ISpanResizer resizer = default) + { + this.resizer = resizer ?? DefaultSpanResizer.Default; + this.buffer = this.resizer.Resize(capacity); + this.length = 0; + this.resolver = null; + } + + /// Initializes a new instance of the struct from an existing buffer. + /// + /// The buffer. The row takes ownership of the buffer and the caller should not + /// maintain a pointer or mutate the buffer after this call returns. + /// + /// The version of the Hybrid Row format to used to encoding the buffer. + /// The resolver for UDTs. + /// Optional memory resizer. + public RowBuffer(Span buffer, HybridRowVersion version, LayoutResolver resolver, ISpanResizer resizer = default) + { + Contract.Requires(buffer.Length >= HybridRowHeader.Size); + this.resizer = resizer ?? DefaultSpanResizer.Default; + this.length = buffer.Length; + this.buffer = buffer; + this.resolver = resolver; + + HybridRowHeader header = this.ReadHeader(0); + Contract.Invariant(header.Version == version); + Layout layout = resolver.Resolve(header.SchemaId); + Contract.Assert(header.SchemaId == layout.SchemaId); + Contract.Invariant(HybridRowHeader.Size + layout.Size <= this.length); + } + + /// The root header for the row. + public HybridRowHeader Header => this.ReadHeader(0); + + /// The length of row in bytes. + public int Length => this.length; + + /// The length of row in bytes. + public byte[] ToArray() + { + return this.buffer.Slice(0, this.length).ToArray(); + } + + /// The resolver for UDTs. + public LayoutResolver Resolver => this.resolver; + + /// Clears all content from the row. The row is empty after this method. + public void Reset() + { + this.length = 0; + this.resolver = null; + } + + /// Copies the content of the buffer into the target stream. + public void WriteTo(Stream stream) + { + stream.Write(this.buffer.Slice(0, this.length)); + } + + /// + /// Reads in the contents of the RowBuffer from an input stream and initializes the row buffer + /// with the associated layout and rowVersion. + /// + /// true if the serialization succeeded. false if the input stream was corrupted. + public bool ReadFrom( + Stream inputStream, + int bytesCount, + HybridRowVersion rowVersion, + LayoutResolver resolver) + { + Contract.Requires(inputStream != null); + Contract.Assert(bytesCount >= HybridRowHeader.Size); + + this.Reset(); + this.resolver = resolver; + this.Ensure(bytesCount); + Contract.Assert(this.buffer.Length >= bytesCount); + this.length = bytesCount; + Span active = this.buffer.Slice(0, bytesCount); + int bytesRead; + do + { + bytesRead = inputStream.Read(active); + active = active.Slice(bytesRead); + } + while (bytesRead != 0); + + if (active.Length != 0) + { + return false; + } + + return this.InitReadFrom(rowVersion); + } + + /// + /// Reads in the contents of the RowBuffer from an existing block of memory and initializes + /// the row buffer with the associated layout and rowVersion. + /// + /// true if the serialization succeeded. false if the input stream was corrupted. + public bool ReadFrom(ReadOnlySpan input, HybridRowVersion rowVersion, LayoutResolver resolver) + { + int bytesCount = input.Length; + Contract.Assert(bytesCount >= HybridRowHeader.Size); + + this.Reset(); + this.resolver = resolver; + this.Ensure(bytesCount); + Contract.Assert(this.buffer.Length >= bytesCount); + input.CopyTo(this.buffer); + this.length = bytesCount; + return this.InitReadFrom(rowVersion); + } + + /// Initializes a row to the minimal size for the given layout. + /// The version of the Hybrid Row format to use for encoding this row. + /// The layout that describes the column layout of the row. + /// The resolver for UDTs. + /// + /// The row is initialized to default row for the given layout. All fixed columns have their + /// default values. All variable columns are null. No sparse columns are present. The row is valid. + /// + public void InitLayout(HybridRowVersion version, Layout layout, LayoutResolver resolver) + { + Contract.Requires(layout != null); + this.resolver = resolver; + + // Ensure sufficient space for fixed schema fields. + this.Ensure(HybridRowHeader.Size + layout.Size); + this.length = HybridRowHeader.Size + layout.Size; + + // Clear all presence bits. + this.buffer.Slice(HybridRowHeader.Size, layout.Size).Fill(0); + + // Set the header. + this.WriteHeader(0, new HybridRowHeader(version, layout.SchemaId)); + } + + internal void WriteHeader(int offset, HybridRowHeader value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal HybridRowHeader ReadHeader(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteSchemaId(int offset, SchemaId value) + { + this.WriteInt32(offset, value.Id); + } + + internal SchemaId ReadSchemaId(int offset) + { + return new SchemaId(this.ReadInt32(offset)); + } + + internal void SetBit(int offset, LayoutBit bit) + { + if (bit.IsInvalid) + { + return; + } + + this.buffer[bit.GetOffset(offset)] |= unchecked((byte)(1 << bit.GetBit())); + } + + internal void UnsetBit(int offset, LayoutBit bit) + { + Contract.Assert(bit != LayoutBit.Invalid); + this.buffer[bit.GetOffset(offset)] &= unchecked((byte)~(1 << bit.GetBit())); + } + + internal bool ReadBit(int offset, LayoutBit bit) + { + if (bit.IsInvalid) + { + return true; + } + + return (this.buffer[bit.GetOffset(offset)] & unchecked((byte)(1 << bit.GetBit()))) != 0; + } + + internal void DeleteVariable(int offset, bool isVarint) + { + ulong existingValueBytes = this.Read7BitEncodedUInt(offset, out int spaceAvailable); + if (!isVarint) + { + spaceAvailable += (int)existingValueBytes; // "size" already in spaceAvailable + } + + this.buffer.Slice(offset + spaceAvailable, this.length - (offset + spaceAvailable)).CopyTo(this.buffer.Slice(offset)); + this.length -= spaceAvailable; + } + + internal void WriteInt8(int offset, sbyte value) + { + this.buffer[offset] = unchecked((byte)value); + } + + internal sbyte ReadInt8(int offset) + { + return unchecked((sbyte)this.buffer[offset]); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void WriteUInt8(int offset, byte value) + { + this.buffer[offset] = value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal byte ReadUInt8(int offset) + { + return this.buffer[offset]; + } + + internal void WriteInt16(int offset, short value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal short ReadInt16(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteUInt16(int offset, ushort value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal ushort ReadUInt16(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteInt32(int offset, int value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal int ReadInt32(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void IncrementUInt32(int offset, uint increment) + { + MemoryMarshal.Cast(this.buffer.Slice(offset))[0] += increment; + } + + internal void DecrementUInt32(int offset, uint decrement) + { + MemoryMarshal.Cast(this.buffer.Slice(offset))[0] -= decrement; + } + + internal void WriteUInt32(int offset, uint value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal uint ReadUInt32(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteInt64(int offset, long value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal long ReadInt64(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteUInt64(int offset, ulong value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal ulong ReadUInt64(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal int Write7BitEncodedUInt(int offset, ulong value) + { + // Write out an unsigned long 7 bits at a time. The high bit of the byte, + // when set, indicates there are more bytes. + int i = 0; + while (value >= 0x80) + { + this.WriteUInt8(offset + i++, unchecked((byte)(value | 0x80))); + value >>= 7; + } + + this.WriteUInt8(offset + i++, (byte)value); + return i; + } + + internal ulong Read7BitEncodedUInt(int offset, out int lenInBytes) + { + // Read out an unsigned long 7 bits at a time. The high bit of the byte, + // when set, indicates there are more bytes. + ulong b = this.buffer[offset]; + if (b < 0x80) + { + lenInBytes = 1; + return b; + } + + ulong retval = b & 0x7F; + int shift = 7; + do + { + Contract.Assert(shift < 10 * 7); + b = this.buffer[++offset]; + retval |= (b & 0x7F) << shift; + shift += 7; + } + while (b >= 0x80); + + lenInBytes = shift / 7; + return retval; + } + + internal int Write7BitEncodedInt(int offset, long value) + { + return this.Write7BitEncodedUInt(offset, RowBuffer.RotateSignToLsb(value)); + } + + internal long Read7BitEncodedInt(int offset, out int lenInBytes) + { + return RowBuffer.RotateSignToMsb(this.Read7BitEncodedUInt(offset, out lenInBytes)); + } + + internal void WriteFloat32(int offset, float value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal float ReadFloat32(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteFloat64(int offset, double value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal double ReadFloat64(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteFloat128(int offset, Float128 value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal Float128 ReadFloat128(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteDecimal(int offset, decimal value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal decimal ReadDecimal(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteDateTime(int offset, DateTime value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal DateTime ReadDateTime(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteUnixDateTime(int offset, UnixDateTime value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal UnixDateTime ReadUnixDateTime(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteGuid(int offset, Guid value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal Guid ReadGuid(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal void WriteMongoDbObjectId(int offset, MongoDbObjectId value) + { + MemoryMarshal.Write(this.buffer.Slice(offset), ref value); + } + + internal MongoDbObjectId ReadMongoDbObjectId(int offset) + { + return MemoryMarshal.Read(this.buffer.Slice(offset)); + } + + internal Utf8Span ReadFixedString(int offset, int len) + { + return Utf8Span.UnsafeFromUtf8BytesNoValidation(this.buffer.Slice(offset, len)); + } + + internal void WriteFixedString(int offset, Utf8Span value) + { + value.Span.CopyTo(this.buffer.Slice(offset)); + } + + internal ReadOnlySpan ReadFixedBinary(int offset, int len) + { + return this.buffer.Slice(offset, len); + } + + internal void WriteFixedBinary(int offset, ReadOnlySpan value, int len) + { + value.CopyTo(this.buffer.Slice(offset, len)); + if (value.Length < len) + { + this.buffer.Slice(offset + value.Length, len - value.Length).Fill(0); + } + } + + internal void WriteFixedBinary(int offset, ReadOnlySequence value, int len) + { + value.CopyTo(this.buffer.Slice(offset, len)); + if (value.Length < len) + { + this.buffer.Slice(offset + (int)value.Length, len - (int)value.Length).Fill(0); + } + } + + internal Utf8Span ReadVariableString(int offset) + { + return this.ReadString(offset, out int _); + } + + internal void WriteVariableString(int offset, Utf8Span value, bool exists, out int shift) + { + int numBytes = value.Length; + this.EnsureVariable(offset, false, numBytes, exists, out int spaceNeeded, out shift); + + int sizeLenInBytes = this.WriteString(offset, value); + Contract.Assert(spaceNeeded == numBytes + sizeLenInBytes); + this.length += shift; + } + + internal ReadOnlySpan ReadVariableBinary(int offset) + { + return this.ReadBinary(offset, out int _); + } + + internal void WriteVariableBinary(int offset, ReadOnlySpan value, bool exists, out int shift) + { + int numBytes = value.Length; + this.EnsureVariable(offset, false, numBytes, exists, out int spaceNeeded, out shift); + + int sizeLenInBytes = this.WriteBinary(offset, value); + Contract.Assert(spaceNeeded == numBytes + sizeLenInBytes); + this.length += shift; + } + + internal void WriteVariableBinary(int offset, ReadOnlySequence value, bool exists, out int shift) + { + int numBytes = (int)value.Length; + this.EnsureVariable(offset, false, numBytes, exists, out int spaceNeeded, out shift); + + int sizeLenInBytes = this.WriteBinary(offset, value); + Contract.Assert(spaceNeeded == numBytes + sizeLenInBytes); + this.length += shift; + } + + internal long ReadVariableInt(int offset) + { + return this.Read7BitEncodedInt(offset, out int _); + } + + internal void WriteVariableInt(int offset, long value, bool exists, out int shift) + { + int numBytes = RowBuffer.Count7BitEncodedInt(value); + this.EnsureVariable(offset, true, numBytes, exists, out int spaceNeeded, out shift); + + int sizeLenInBytes = this.Write7BitEncodedInt(offset, value); + Contract.Assert(sizeLenInBytes == numBytes); + Contract.Assert(spaceNeeded == numBytes); + this.length += shift; + } + + internal ulong ReadVariableUInt(int offset) + { + return this.Read7BitEncodedUInt(offset, out int _); + } + + internal void WriteVariableUInt(int offset, ulong value, bool exists, out int shift) + { + int numBytes = RowBuffer.Count7BitEncodedUInt(value); + this.EnsureVariable(offset, true, numBytes, exists, out int spaceNeeded, out shift); + + int sizeLenInBytes = this.Write7BitEncodedUInt(offset, value); + Contract.Assert(sizeLenInBytes == numBytes); + Contract.Assert(spaceNeeded == numBytes); + this.length += shift; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal LayoutType ReadSparseTypeCode(int offset) + { + return LayoutType.FromCode((LayoutCode)this.ReadUInt8(offset)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void WriteSparseTypeCode(int offset, LayoutCode code) + { + this.WriteUInt8(offset, (byte)code); + } + + internal sbyte ReadSparseInt8(ref RowCursor edit) + { + // TODO: Remove calls to ReadSparsePrimitiveTypeCode once moved to V2 read. + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Int8); + edit.endOffset = edit.valueOffset + sizeof(sbyte); + return this.ReadInt8(edit.valueOffset); + } + + internal void WriteSparseInt8(ref RowCursor edit, sbyte value, UpdateOptions options) + { + int numBytes = sizeof(sbyte); + this.EnsureSparse( + ref edit, + LayoutType.Int8, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + + this.WriteSparseMetadata(ref edit, LayoutType.Int8, TypeArgumentList.Empty, metaBytes); + this.WriteInt8(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(sbyte)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal short ReadSparseInt16(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Int16); + edit.endOffset = edit.valueOffset + sizeof(short); + return this.ReadInt16(edit.valueOffset); + } + + internal void WriteSparseInt16(ref RowCursor edit, short value, UpdateOptions options) + { + int numBytes = sizeof(short); + this.EnsureSparse( + ref edit, + LayoutType.Int16, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Int16, TypeArgumentList.Empty, metaBytes); + this.WriteInt16(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(short)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal int ReadSparseInt32(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Int32); + edit.endOffset = edit.valueOffset + sizeof(int); + return this.ReadInt32(edit.valueOffset); + } + + internal void WriteSparseInt32(ref RowCursor edit, int value, UpdateOptions options) + { + int numBytes = sizeof(int); + this.EnsureSparse( + ref edit, + LayoutType.Int32, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Int32, TypeArgumentList.Empty, metaBytes); + this.WriteInt32(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(int)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal long ReadSparseInt64(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Int64); + edit.endOffset = edit.valueOffset + sizeof(long); + return this.ReadInt64(edit.valueOffset); + } + + internal void WriteSparseInt64(ref RowCursor edit, long value, UpdateOptions options) + { + int numBytes = sizeof(long); + this.EnsureSparse( + ref edit, + LayoutType.Int64, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Int64, TypeArgumentList.Empty, metaBytes); + this.WriteInt64(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(long)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal byte ReadSparseUInt8(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.UInt8); + edit.endOffset = edit.valueOffset + sizeof(byte); + return this.ReadUInt8(edit.valueOffset); + } + + internal void WriteSparseUInt8(ref RowCursor edit, byte value, UpdateOptions options) + { + int numBytes = sizeof(byte); + this.EnsureSparse( + ref edit, + LayoutType.UInt8, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.UInt8, TypeArgumentList.Empty, metaBytes); + this.WriteUInt8(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(byte)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal ushort ReadSparseUInt16(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.UInt16); + edit.endOffset = edit.valueOffset + sizeof(ushort); + return this.ReadUInt16(edit.valueOffset); + } + + internal void WriteSparseUInt16(ref RowCursor edit, ushort value, UpdateOptions options) + { + int numBytes = sizeof(ushort); + this.EnsureSparse( + ref edit, + LayoutType.UInt16, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.UInt16, TypeArgumentList.Empty, metaBytes); + this.WriteUInt16(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(ushort)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal uint ReadSparseUInt32(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.UInt32); + edit.endOffset = edit.valueOffset + sizeof(uint); + return this.ReadUInt32(edit.valueOffset); + } + + internal void WriteSparseUInt32(ref RowCursor edit, uint value, UpdateOptions options) + { + int numBytes = sizeof(uint); + this.EnsureSparse( + ref edit, + LayoutType.UInt32, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.UInt32, TypeArgumentList.Empty, metaBytes); + this.WriteUInt32(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(uint)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal ulong ReadSparseUInt64(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.UInt64); + edit.endOffset = edit.valueOffset + sizeof(ulong); + return this.ReadUInt64(edit.valueOffset); + } + + internal void WriteSparseUInt64(ref RowCursor edit, ulong value, UpdateOptions options) + { + int numBytes = sizeof(ulong); + this.EnsureSparse( + ref edit, + LayoutType.UInt64, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.UInt64, TypeArgumentList.Empty, metaBytes); + this.WriteUInt64(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(ulong)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal long ReadSparseVarInt(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.VarInt); + long value = this.Read7BitEncodedInt(edit.valueOffset, out int sizeLenInBytes); + edit.endOffset = edit.valueOffset + sizeLenInBytes; + return value; + } + + internal void WriteSparseVarInt(ref RowCursor edit, long value, UpdateOptions options) + { + int numBytes = RowBuffer.Count7BitEncodedInt(value); + this.EnsureSparse( + ref edit, + LayoutType.VarInt, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.VarInt, TypeArgumentList.Empty, metaBytes); + int sizeLenInBytes = this.Write7BitEncodedInt(edit.valueOffset, value); + Contract.Assert(sizeLenInBytes == numBytes); + Contract.Assert(spaceNeeded == metaBytes + sizeLenInBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal ulong ReadSparseVarUInt(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.VarUInt); + ulong value = this.Read7BitEncodedUInt(edit.valueOffset, out int sizeLenInBytes); + edit.endOffset = edit.valueOffset + sizeLenInBytes; + return value; + } + + internal void WriteSparseVarUInt(ref RowCursor edit, ulong value, UpdateOptions options) + { + int numBytes = RowBuffer.Count7BitEncodedUInt(value); + this.EnsureSparse( + ref edit, + LayoutType.VarUInt, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.VarUInt, TypeArgumentList.Empty, metaBytes); + int sizeLenInBytes = this.Write7BitEncodedUInt(edit.valueOffset, value); + Contract.Assert(sizeLenInBytes == numBytes); + Contract.Assert(spaceNeeded == metaBytes + sizeLenInBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal float ReadSparseFloat32(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Float32); + edit.endOffset = edit.valueOffset + sizeof(float); + return this.ReadFloat32(edit.valueOffset); + } + + internal void WriteSparseFloat32(ref RowCursor edit, float value, UpdateOptions options) + { + int numBytes = sizeof(float); + this.EnsureSparse( + ref edit, + LayoutType.Float32, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Float32, TypeArgumentList.Empty, metaBytes); + this.WriteFloat32(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(float)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal double ReadSparseFloat64(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Float64); + edit.endOffset = edit.valueOffset + sizeof(double); + return this.ReadFloat64(edit.valueOffset); + } + + internal void WriteSparseFloat64(ref RowCursor edit, double value, UpdateOptions options) + { + int numBytes = sizeof(double); + this.EnsureSparse( + ref edit, + LayoutType.Float64, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Float64, TypeArgumentList.Empty, metaBytes); + this.WriteFloat64(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(double)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal Float128 ReadSparseFloat128(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Float128); + edit.endOffset = edit.valueOffset + Float128.Size; + return this.ReadFloat128(edit.valueOffset); + } + + internal void WriteSparseFloat128(ref RowCursor edit, Float128 value, UpdateOptions options) + { + int numBytes = Float128.Size; + this.EnsureSparse( + ref edit, + LayoutType.Float128, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Float128, TypeArgumentList.Empty, metaBytes); + this.WriteFloat128(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + Float128.Size); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal decimal ReadSparseDecimal(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Decimal); + edit.endOffset = edit.valueOffset + sizeof(decimal); + return this.ReadDecimal(edit.valueOffset); + } + + internal void WriteSparseDecimal(ref RowCursor edit, decimal value, UpdateOptions options) + { + int numBytes = sizeof(decimal); + this.EnsureSparse( + ref edit, + LayoutType.Decimal, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Decimal, TypeArgumentList.Empty, metaBytes); + this.WriteDecimal(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + sizeof(decimal)); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal DateTime ReadSparseDateTime(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.DateTime); + edit.endOffset = edit.valueOffset + 8; + return this.ReadDateTime(edit.valueOffset); + } + + internal void WriteSparseDateTime(ref RowCursor edit, DateTime value, UpdateOptions options) + { + int numBytes = 8; + this.EnsureSparse( + ref edit, + LayoutType.DateTime, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.DateTime, TypeArgumentList.Empty, metaBytes); + this.WriteDateTime(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + 8); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal UnixDateTime ReadSparseUnixDateTime(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.UnixDateTime); + edit.endOffset = edit.valueOffset + 8; + return this.ReadUnixDateTime(edit.valueOffset); + } + + internal void WriteSparseUnixDateTime(ref RowCursor edit, UnixDateTime value, UpdateOptions options) + { + int numBytes = 8; + this.EnsureSparse( + ref edit, + LayoutType.UnixDateTime, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + + this.WriteSparseMetadata(ref edit, LayoutType.UnixDateTime, TypeArgumentList.Empty, metaBytes); + this.WriteUnixDateTime(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + 8); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal Guid ReadSparseGuid(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Guid); + edit.endOffset = edit.valueOffset + 16; + return this.ReadGuid(edit.valueOffset); + } + + internal void WriteSparseGuid(ref RowCursor edit, Guid value, UpdateOptions options) + { + int numBytes = 16; + this.EnsureSparse( + ref edit, + LayoutType.Guid, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Guid, TypeArgumentList.Empty, metaBytes); + this.WriteGuid(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + 16); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal MongoDbObjectId ReadSparseMongoDbObjectId(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.MongoDbObjectId); + edit.endOffset = edit.valueOffset + MongoDbObjectId.Size; + return this.ReadMongoDbObjectId(edit.valueOffset); + } + + internal void WriteSparseMongoDbObjectId(ref RowCursor edit, MongoDbObjectId value, UpdateOptions options) + { + int numBytes = MongoDbObjectId.Size; + this.EnsureSparse( + ref edit, + LayoutType.MongoDbObjectId, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + + this.WriteSparseMetadata(ref edit, LayoutType.MongoDbObjectId, TypeArgumentList.Empty, metaBytes); + this.WriteMongoDbObjectId(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + MongoDbObjectId.Size); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal NullValue ReadSparseNull(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Null); + edit.endOffset = edit.valueOffset; + return NullValue.Default; + } + + internal void WriteSparseNull(ref RowCursor edit, NullValue value, UpdateOptions options) + { + int numBytes = 0; + this.EnsureSparse( + ref edit, + LayoutType.Null, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Null, TypeArgumentList.Empty, metaBytes); + Contract.Assert(spaceNeeded == metaBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal bool ReadSparseBool(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Boolean); + edit.endOffset = edit.valueOffset; + return edit.cellType == LayoutType.Boolean; + } + + internal void WriteSparseBool(ref RowCursor edit, bool value, UpdateOptions options) + { + int numBytes = 0; + this.EnsureSparse( + ref edit, + value ? LayoutType.Boolean : LayoutType.BooleanFalse, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, value ? LayoutType.Boolean : LayoutType.BooleanFalse, TypeArgumentList.Empty, metaBytes); + Contract.Assert(spaceNeeded == metaBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal Utf8Span ReadSparseString(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Utf8); + Utf8Span span = this.ReadString(edit.valueOffset, out int sizeLenInBytes); + edit.endOffset = edit.valueOffset + sizeLenInBytes + span.Length; + return span; + } + + internal void WriteSparseString(ref RowCursor edit, Utf8Span value, UpdateOptions options) + { + int len = value.Length; + int numBytes = len + RowBuffer.Count7BitEncodedUInt((ulong)len); + this.EnsureSparse( + ref edit, + LayoutType.Utf8, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Utf8, TypeArgumentList.Empty, metaBytes); + int sizeLenInBytes = this.WriteString(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + len + sizeLenInBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal ReadOnlySpan ReadSparseBinary(ref RowCursor edit) + { + this.ReadSparsePrimitiveTypeCode(ref edit, LayoutType.Binary); + ReadOnlySpan span = this.ReadBinary(edit.valueOffset, out int sizeLenInBytes); + edit.endOffset = edit.valueOffset + sizeLenInBytes + span.Length; + return span; + } + + internal void WriteSparseBinary(ref RowCursor edit, ReadOnlySpan value, UpdateOptions options) + { + int len = value.Length; + int numBytes = len + RowBuffer.Count7BitEncodedUInt((ulong)len); + this.EnsureSparse( + ref edit, + LayoutType.Binary, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Binary, TypeArgumentList.Empty, metaBytes); + int sizeLenInBytes = this.WriteBinary(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + len + sizeLenInBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal void WriteSparseBinary(ref RowCursor edit, ReadOnlySequence value, UpdateOptions options) + { + int len = (int)value.Length; + int numBytes = len + RowBuffer.Count7BitEncodedUInt((ulong)len); + this.EnsureSparse( + ref edit, + LayoutType.Binary, + TypeArgumentList.Empty, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shift); + this.WriteSparseMetadata(ref edit, LayoutType.Binary, TypeArgumentList.Empty, metaBytes); + int sizeLenInBytes = this.WriteBinary(edit.valueOffset, value); + Contract.Assert(spaceNeeded == metaBytes + len + sizeLenInBytes); + edit.endOffset = edit.metaOffset + spaceNeeded; + this.length += shift; + } + + internal void WriteSparseObject(ref RowCursor edit, LayoutScope scopeType, UpdateOptions options, out RowCursor newScope) + { + int numBytes = sizeof(LayoutCode); // end scope type code. + TypeArgumentList typeArgs = TypeArgumentList.Empty; + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, TypeArgumentList.Empty, metaBytes); + this.WriteSparseTypeCode(edit.valueOffset, LayoutCode.EndScope); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = TypeArgumentList.Empty, + start = edit.valueOffset, + valueOffset = edit.valueOffset, + metaOffset = edit.valueOffset, + layout = edit.layout, + }; + + this.length += shift; + } + + internal void WriteSparseArray(ref RowCursor edit, LayoutScope scopeType, UpdateOptions options, out RowCursor newScope) + { + int numBytes = sizeof(LayoutCode); // end scope type code. + TypeArgumentList typeArgs = TypeArgumentList.Empty; + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + this.WriteSparseTypeCode(edit.valueOffset, LayoutCode.EndScope); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, + valueOffset = edit.valueOffset, + metaOffset = edit.valueOffset, + layout = edit.layout, + }; + + this.length += shift; + } + + internal void WriteTypedArray( + ref RowCursor edit, + LayoutScope scopeType, + TypeArgumentList typeArgs, + UpdateOptions options, + out RowCursor newScope) + { + int numBytes = sizeof(uint); + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + this.WriteUInt32(edit.valueOffset, 0); + int valueOffset = edit.valueOffset + sizeof(uint); // Point after the Size + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, // Point at the Size + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + }; + + this.length += shift; + } + + internal void WriteTypedSet( + ref RowCursor edit, + LayoutScope scopeType, + TypeArgumentList typeArgs, + UpdateOptions options, + out RowCursor newScope) + { + int numBytes = sizeof(uint); + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + this.WriteUInt32(edit.valueOffset, 0); + int valueOffset = edit.valueOffset + sizeof(uint); // Point after the Size + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, // Point at the Size + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + }; + + this.length += shift; + } + + internal void WriteTypedMap( + ref RowCursor edit, + LayoutScope scopeType, + TypeArgumentList typeArgs, + UpdateOptions options, + out RowCursor newScope) + { + int numBytes = sizeof(uint); // Sized scope. + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + this.WriteUInt32(edit.valueOffset, 0); + int valueOffset = edit.valueOffset + sizeof(uint); // Point after the Size + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, // Point at the Size + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + }; + + this.length += shift; + } + + internal void WriteSparseTuple( + ref RowCursor edit, + LayoutScope scopeType, + TypeArgumentList typeArgs, + UpdateOptions options, + out RowCursor newScope) + { + int numBytes = sizeof(LayoutCode) * (1 + typeArgs.Count); // nulls for each element. + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + int valueOffset = edit.valueOffset; + for (int i = 0; i < typeArgs.Count; i++) + { + this.WriteSparseTypeCode(valueOffset, LayoutCode.Null); + valueOffset += sizeof(LayoutCode); + } + + this.WriteSparseTypeCode(valueOffset, LayoutCode.EndScope); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, + valueOffset = edit.valueOffset, + metaOffset = edit.valueOffset, + layout = edit.layout, + count = typeArgs.Count, + }; + + this.length += shift; + } + + internal void WriteTypedTuple( + ref RowCursor edit, + LayoutScope scopeType, + TypeArgumentList typeArgs, + UpdateOptions options, + out RowCursor newScope) + { + int numBytes = this.CountDefaultValue(scopeType, typeArgs); + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + int numWritten = this.WriteDefaultValue(edit.valueOffset, scopeType, typeArgs); + Contract.Assert(numBytes == numWritten); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, + valueOffset = edit.valueOffset, + metaOffset = edit.valueOffset, + layout = edit.layout, + count = typeArgs.Count, + }; + + this.length += shift; + newScope.MoveNext(ref this); + } + + internal void WriteNullable( + ref RowCursor edit, + LayoutScope scopeType, + TypeArgumentList typeArgs, + UpdateOptions options, + bool hasValue, + out RowCursor newScope) + { + int numBytes = this.CountDefaultValue(scopeType, typeArgs); + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + int numWritten = this.WriteDefaultValue(edit.valueOffset, scopeType, typeArgs); + Contract.Assert(numBytes == numWritten); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + if (hasValue) + { + this.WriteInt8(edit.valueOffset, 1); + } + + int valueOffset = edit.valueOffset + 1; + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + count = 2, + index = 1, + }; + + this.length += shift; + newScope.MoveNext(ref this); + } + + internal void WriteSparseUDT( + ref RowCursor edit, + LayoutScope scopeType, + Layout udt, + UpdateOptions options, + out RowCursor newScope) + { + TypeArgumentList typeArgs = new TypeArgumentList(udt.SchemaId); + int numBytes = udt.Size + sizeof(LayoutCode); + this.EnsureSparse(ref edit, scopeType, typeArgs, numBytes, options, out int metaBytes, out int spaceNeeded, out int shift); + this.WriteSparseMetadata(ref edit, scopeType, typeArgs, metaBytes); + + // Clear all presence bits. + this.buffer.Slice(edit.valueOffset, udt.Size).Fill(0); + + // Write scope terminator. + int valueOffset = edit.valueOffset + udt.Size; + this.WriteSparseTypeCode(valueOffset, LayoutCode.EndScope); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + newScope = new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = typeArgs, + start = edit.valueOffset, + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = udt, + }; + + this.length += shift; + } + + /// Delete the sparse field at the indicated path. + /// The field to delete. + internal void DeleteSparse(ref RowCursor edit) + { + // If the field doesn't exist, then nothing to do. + if (!edit.exists) + { + return; + } + + int numBytes = 0; + this.EnsureSparse( + ref edit, + edit.cellType, + edit.cellTypeArgs, + numBytes, + RowOptions.Delete, + out int _, + out int _, + out int shift); + this.length += shift; + } + + /// Rotates the sign bit of a two's complement value to the least significant bit. + /// A signed value. + /// An unsigned value encoding the same value but with the sign bit in the LSB. + /// + /// Moves the signed bit of a two's complement value to the least significant bit (LSB) by: + /// + /// + /// If negative, take the two's complement. + /// + /// Left shift the value by 1 bit. + /// + /// If negative, set the LSB to 1. + /// + /// + /// + [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1629:DocumentationTextMustEndWithAPeriod", Justification = "Colon.")] + internal static ulong RotateSignToLsb(long value) + { + // Rotate sign to LSB + unchecked + { + bool isNegative = value < 0; + ulong uvalue = (ulong)value; + uvalue = isNegative ? ((~uvalue + 1ul) << 1) + 1ul : uvalue << 1; + return uvalue; + } + } + + /// Undoes the rotation introduced by . + /// An unsigned value with the sign bit in the LSB. + /// A signed two's complement value encoding the same value. + internal static long RotateSignToMsb(ulong uvalue) + { + // Rotate sign to MSB + unchecked + { + bool isNegative = uvalue % 2 != 0; + long value = (long)(isNegative ? (~(uvalue >> 1) + 1ul) | 0x8000000000000000ul : uvalue >> 1); + return value; + } + } + + /// Compute the byte offset from the beginning of the row for a given variable column's value. + /// The (optional) layout of the current scope. + /// The 0-based offset to the beginning of the scope's value. + /// The 0-based index of the variable column within the variable segment. + /// + /// The byte offset from the beginning of the row where the variable column's value should be + /// located. + /// + internal int ComputeVariableValueOffset(Layout layout, int scopeOffset, int varIndex) + { + if (layout == null) + { + return scopeOffset; + } + + int index = layout.NumFixed + varIndex; + ReadOnlySpan columns = layout.Columns; + Contract.Assert(index <= columns.Length); + int offset = scopeOffset + layout.Size; + for (int i = layout.NumFixed; i < index; i++) + { + LayoutColumn col = columns[i]; + if (this.ReadBit(scopeOffset, col.NullBit)) + { + ulong valueSizeInBytes = this.Read7BitEncodedUInt(offset, out int lengthSizeInBytes); + if (col.Type.IsVarint) + { + offset += lengthSizeInBytes; + } + else + { + offset += (int)valueSizeInBytes + lengthSizeInBytes; + } + } + } + + return offset; + } + + /// Move a sparse iterator to the next field within the same sparse scope. + /// The iterator to advance. + /// + /// + /// On success, the path of the field at the given offset, otherwise + /// undefined. + /// + /// + /// If found, the offset to the metadata of the field, otherwise a + /// location to insert the field. + /// + /// + /// If found, the layout code of the matching field found, otherwise + /// undefined. + /// + /// + /// If found, the offset to the value of the field, otherwise + /// undefined. + /// . + /// + /// True if there is another field, false if there are no more. + internal bool SparseIteratorMoveNext(ref RowCursor edit) + { + if (edit.cellType != null) + { + // Move to the next element of an indexed scope. + if (edit.scopeType.IsIndexedScope) + { + edit.index++; + } + + // Skip forward to the end of the current value. + if (edit.endOffset != 0) + { + edit.metaOffset = edit.endOffset; + edit.endOffset = 0; + } + else + { + edit.metaOffset += this.SparseComputeSize(ref edit); + } + } + + // Check if reached end of buffer. + if (edit.metaOffset < this.length) + { + // Check if reached end of sized scope. + if (!edit.scopeType.IsSizedScope || (edit.index != edit.count)) + { + // Read the metadata. + this.ReadSparseMetadata(ref edit); + + // Check if reached end of sparse scope. + if (!(edit.cellType is LayoutEndScope)) + { + edit.exists = true; + return true; + } + } + } + + edit.cellType = LayoutType.EndScope; + edit.exists = false; + edit.valueOffset = edit.metaOffset; + return false; + } + + /// Produce a new scope from the current iterator position. + /// An initialized iterator pointing at a scope. + /// True if the new scope should be marked immutable (read-only). + /// A new scope beginning at the current iterator position. + internal RowCursor SparseIteratorReadScope(ref RowCursor edit, bool immutable) + { + LayoutScope scopeType = edit.cellType as LayoutScope; + switch (scopeType) + { + case LayoutObject _: + case LayoutArray _: + { + return new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = edit.cellTypeArgs, + start = edit.valueOffset, + valueOffset = edit.valueOffset, + metaOffset = edit.valueOffset, + layout = edit.layout, + immutable = immutable, + }; + + break; + } + + case LayoutTypedArray _: + case LayoutTypedSet _: + case LayoutTypedMap _: + { + int valueOffset = edit.valueOffset + sizeof(uint); // Point after the Size + return new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = edit.cellTypeArgs, + start = edit.valueOffset, // Point at the Size + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + immutable = immutable, + count = (int)this.ReadUInt32(edit.valueOffset), + }; + + break; + } + + case LayoutTypedTuple _: + case LayoutTuple _: + case LayoutTagged _: + case LayoutTagged2 _: + { + return new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = edit.cellTypeArgs, + start = edit.valueOffset, + valueOffset = edit.valueOffset, + metaOffset = edit.valueOffset, + layout = edit.layout, + immutable = immutable, + count = edit.cellTypeArgs.Count, + }; + + break; + } + + case LayoutNullable _: + { + bool hasValue = this.ReadInt8(edit.valueOffset) != 0; + if (hasValue) + { + // Start at the T so it can be read. + int valueOffset = edit.valueOffset + 1; + return new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = edit.cellTypeArgs, + start = edit.valueOffset, + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + immutable = immutable, + count = 2, + index = 1, + }; + } + else + { + // Start at the end of the scope, instead of at the T, so the T will be skipped. + TypeArgument typeArg = edit.cellTypeArgs[0]; + int valueOffset = edit.valueOffset + 1 + this.CountDefaultValue(typeArg.Type, typeArg.TypeArgs); + return new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = edit.cellTypeArgs, + start = edit.valueOffset, + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = edit.layout, + immutable = immutable, + count = 2, + index = 2, + }; + } + + break; + } + + case LayoutUDT _: + { + Layout udt = this.resolver.Resolve(edit.cellTypeArgs.SchemaId); + int valueOffset = this.ComputeVariableValueOffset(udt, edit.valueOffset, udt.NumVariable); + return new RowCursor + { + scopeType = scopeType, + scopeTypeArgs = edit.cellTypeArgs, + start = edit.valueOffset, + valueOffset = valueOffset, + metaOffset = valueOffset, + layout = udt, + immutable = immutable, + }; + + break; + } + + default: + Contract.Fail("Not a scope type."); + return default; + } + } + + /// + /// Compute the byte offsets from the beginning of the row for a given sparse field insertion + /// into a set/map. + /// + /// The sparse scope to insert into. + /// The field to move into the set/map. + /// The prepared edit context. + internal RowCursor PrepareSparseMove(ref RowCursor scope, ref RowCursor srcEdit) + { + Contract.Requires(scope.scopeType.IsUniqueScope); + + Contract.Requires(scope.index == 0); + scope.Clone(out RowCursor dstEdit); + + dstEdit.metaOffset = scope.valueOffset; + int srcSize = this.SparseComputeSize(ref srcEdit); + int srcBytes = srcSize - (srcEdit.valueOffset - srcEdit.metaOffset); + while (dstEdit.index < dstEdit.count) + { + this.ReadSparseMetadata(ref dstEdit); + Contract.Assert(dstEdit.pathOffset == default); + + int elmSize = -1; // defer calculating the full size until needed. + int cmp; + if (scope.scopeType is LayoutTypedMap) + { + cmp = this.CompareKeyValueFieldValue(srcEdit, dstEdit); + } + else + { + elmSize = this.SparseComputeSize(ref dstEdit); + int elmBytes = elmSize - (dstEdit.valueOffset - dstEdit.metaOffset); + cmp = this.CompareFieldValue(srcEdit, srcBytes, dstEdit, elmBytes); + } + + if (cmp <= 0) + { + dstEdit.exists = cmp == 0; + return dstEdit; + } + + elmSize = (elmSize == -1) ? this.SparseComputeSize(ref dstEdit) : elmSize; + dstEdit.index++; + dstEdit.metaOffset += elmSize; + } + + dstEdit.exists = false; + dstEdit.cellType = LayoutType.EndScope; + dstEdit.valueOffset = dstEdit.metaOffset; + return dstEdit; + } + + internal void TypedCollectionMoveField(ref RowCursor dstEdit, ref RowCursor srcEdit, RowOptions options) + { + int encodedSize = this.SparseComputeSize(ref srcEdit); + int numBytes = encodedSize - (srcEdit.valueOffset - srcEdit.metaOffset); + + // Insert the field metadata into its new location. + this.EnsureSparse( + ref dstEdit, + srcEdit.cellType, + srcEdit.cellTypeArgs, + numBytes, + options, + out int metaBytes, + out int spaceNeeded, + out int shiftInsert); + + this.WriteSparseMetadata(ref dstEdit, srcEdit.cellType, srcEdit.cellTypeArgs, metaBytes); + Contract.Assert(spaceNeeded == metaBytes + numBytes); + if (srcEdit.metaOffset >= dstEdit.metaOffset) + { + srcEdit.metaOffset += shiftInsert; + srcEdit.valueOffset += shiftInsert; + } + + // Copy the value bits from the old location. + this.buffer.Slice(srcEdit.valueOffset, numBytes).CopyTo(this.buffer.Slice(dstEdit.valueOffset)); + this.length += shiftInsert; + + // Delete the old location. + this.EnsureSparse( + ref srcEdit, + srcEdit.cellType, + srcEdit.cellTypeArgs, + numBytes, + RowOptions.Delete, + out metaBytes, + out spaceNeeded, + out int shiftDelete); + + Contract.Assert(shiftDelete < 0); + this.length += shiftDelete; + } + + /// Rebuild the unique index for a set/map scope. + /// The sparse scope to rebuild an index for. + /// Success if the index could be built, an error otherwise. + /// + /// The MUST be a set or map scope. + /// + /// The scope may have been built (e.g. via RowWriter) with relaxed uniqueness constraint checking. + /// This operation rebuilds an index to support verification of uniqueness constraints during + /// subsequent partial updates. If the appropriate uniqueness constraints cannot be established (i.e. + /// a duplicate exists), this operation fails. Before continuing, the resulting scope should either: + /// + /// + /// + /// Be repaired (e.g. by deleting duplicates) and the index rebuild operation should be + /// run again. + /// + /// + /// Be deleted. The entire scope should be removed including its items. + /// + /// Failure to perform one of these actions will leave the row is potentially in a corrupted + /// state where partial updates may subsequent fail. + /// + /// + /// The target may or may not have already been indexed. This + /// operation is idempotent. + /// + /// + internal Result TypedCollectionUniqueIndexRebuild(ref RowCursor scope) + { + Contract.Requires(scope.scopeType.IsUniqueScope); + Contract.Requires(scope.index == 0); + scope.Clone(out RowCursor dstEdit); + if (dstEdit.count <= 1) + { + return Result.Success; + } + + // Compute Index Elements. + Span uniqueIndex = dstEdit.count < 100 ? stackalloc UniqueIndexItem[dstEdit.count] : new UniqueIndexItem[dstEdit.count]; + dstEdit.metaOffset = scope.valueOffset; + for (; dstEdit.index < dstEdit.count; dstEdit.index++) + { + this.ReadSparseMetadata(ref dstEdit); + Contract.Assert(dstEdit.pathOffset == default); + int elmSize = this.SparseComputeSize(ref dstEdit); + + uniqueIndex[dstEdit.index] = new UniqueIndexItem + { + Code = dstEdit.cellType.LayoutCode, + MetaOffset = dstEdit.metaOffset, + ValueOffset = dstEdit.valueOffset, + Size = elmSize, + }; + + dstEdit.metaOffset += elmSize; + } + + // Create scratch space equal to the sum of the sizes of the scope's values. + // Implementation Note: theoretically this scratch space could be eliminated by + // performing the item move operations directly during the Insertion Sort, however, + // doing so might result in moving the same item multiple times. Under the assumption + // that items are relatively large, using scratch space requires each item to be moved + // AT MOST once. Given that row buffer memory is likely reused, scratch space is + // relatively memory efficient. + int shift = dstEdit.metaOffset - scope.valueOffset; + + // Sort and check for duplicates. + unsafe + { + Span p = new Span(Unsafe.AsPointer(ref uniqueIndex.GetPinnableReference()), uniqueIndex.Length); + if (!this.InsertionSort(ref scope, ref dstEdit, p)) + { + return Result.Exists; + } + } + + // Move elements. + int metaOffset = scope.valueOffset; + this.Ensure(this.length + shift); + this.buffer.Slice(metaOffset, this.length - metaOffset).CopyTo(this.buffer.Slice(metaOffset + shift)); + foreach (UniqueIndexItem x in uniqueIndex) + { + this.buffer.Slice(x.MetaOffset + shift, x.Size).CopyTo(this.buffer.Slice(metaOffset)); + metaOffset += x.Size; + } + + // Delete the scratch space (if necessary - if it doesn't just fall off the end of the row). + if (metaOffset != this.length) + { + this.buffer.Slice(metaOffset + shift, this.length - metaOffset).CopyTo(this.buffer.Slice(metaOffset)); + } + +#if DEBUG + // Fill deleted bits (in debug builds) to detect overflow/alignment errors. + this.buffer.Slice(this.length, shift).Fill(0xFF); +#endif + + return Result.Success; + } + + private static int CountSparsePath(ref RowCursor edit) + { + if (!edit.writePathToken.IsNull) + { + return edit.writePathToken.Varint.Length; + } + + if (edit.layout.Tokenizer.TryFindToken(edit.writePath, out edit.writePathToken)) + { + return edit.writePathToken.Varint.Length; + } + + int numBytes = edit.writePath.ToUtf8String().Length; + int sizeLenInBytes = RowBuffer.Count7BitEncodedUInt((ulong)(edit.layout.Tokenizer.Count + numBytes)); + return sizeLenInBytes + numBytes; + } + + /// + /// Compute the number of bytes necessary to store the unsigned integer using the varuint + /// encoding. + /// + /// The value to be encoded. + /// The number of bytes needed to store the varuint encoding of . + internal static int Count7BitEncodedUInt(ulong value) + { + // Count the number of bytes needed to write out an int 7 bits at a time. + int i = 0; + while (value >= 0x80) + { + i++; + value >>= 7; + } + + i++; + return i; + } + + /// + /// Compute the number of bytes necessary to store the signed integer using the varint + /// encoding. + /// + /// The value to be encoded. + /// The number of bytes needed to store the varint encoding of . + private static int Count7BitEncodedInt(long value) + { + return RowBuffer.Count7BitEncodedUInt(RowBuffer.RotateSignToLsb(value)); + } + + /// + /// Reads in the contents of the RowBuffer from an input stream and initializes the row buffer + /// with the associated layout and rowVersion. + /// + /// true if the serialization succeeded. false if the input stream was corrupted. + private bool InitReadFrom(HybridRowVersion rowVersion) + { + HybridRowHeader header = this.ReadHeader(0); + Layout layout = this.resolver.Resolve(header.SchemaId); + Contract.Assert(header.SchemaId == layout.SchemaId); + if ((header.Version != rowVersion) || (HybridRowHeader.Size + layout.Size > this.length)) + { + return false; + } + + return true; + } + + /// Skip over a nested scope. + /// The sparse scope to search. + /// The 0-based byte offset immediately following the scope end marker. + private int SkipScope(ref RowCursor edit) + { + while (this.SparseIteratorMoveNext(ref edit)) + { + } + + if (!edit.scopeType.IsSizedScope) + { + edit.metaOffset += sizeof(LayoutCode); // Move past the end of scope marker. + } + + return edit.metaOffset; + } + + /// Compares the values of two encoded fields using the hybrid row binary collation. + /// An edit describing the left field. + /// The size of the left field's value in bytes. + /// An edit describing the right field. + /// The size of the right field's value in bytes. + /// + /// + /// + /// -1left less than right. + /// + /// 0left and right are equal. + /// + /// 1left is greater than right. + /// + /// + /// + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1201", Justification = "Logical Grouping.")] + private int CompareFieldValue(RowCursor left, int leftLen, RowCursor right, int rightLen) + { + if (left.cellType.LayoutCode < right.cellType.LayoutCode) + { + return -1; + } + + if (left.cellType == right.cellType) + { + if (leftLen < rightLen) + { + return -1; + } + + if (leftLen == rightLen) + { + return this.buffer.Slice(left.valueOffset, leftLen).SequenceCompareTo(this.buffer.Slice(right.valueOffset, rightLen)); + } + } + + return 1; + } + + /// + /// Compares the values of two encoded key-value pair fields using the hybrid row binary + /// collation. + /// + /// An edit describing the left field. + /// An edit describing the right field. + /// + /// + /// + /// -1left less than right. + /// + /// 0left and right are equal. + /// + /// 1left is greater than right. + /// + /// + /// + private int CompareKeyValueFieldValue(RowCursor left, RowCursor right) + { + LayoutTypedTuple leftScopeType = left.cellType as LayoutTypedTuple; + LayoutTypedTuple rightScopeType = right.cellType as LayoutTypedTuple; + Contract.Requires(leftScopeType != null); + Contract.Requires(rightScopeType != null); + Contract.Requires(left.cellTypeArgs.Count == 2); + Contract.Requires(left.cellTypeArgs.Equals(right.cellTypeArgs)); + + RowCursor leftKey = new RowCursor + { + layout = left.layout, + scopeType = leftScopeType, + scopeTypeArgs = left.cellTypeArgs, + start = left.valueOffset, + metaOffset = left.valueOffset, + index = 0, + }; + + this.ReadSparseMetadata(ref leftKey); + Contract.Assert(leftKey.pathOffset == default); + int leftKeyLen = this.SparseComputeSize(ref leftKey) - (leftKey.valueOffset - leftKey.metaOffset); + + RowCursor rightKey = new RowCursor + { + layout = right.layout, + scopeType = rightScopeType, + scopeTypeArgs = right.cellTypeArgs, + start = right.valueOffset, + metaOffset = right.valueOffset, + index = 0, + }; + + this.ReadSparseMetadata(ref rightKey); + Contract.Assert(rightKey.pathOffset == default); + int rightKeyLen = this.SparseComputeSize(ref rightKey) - (rightKey.valueOffset - rightKey.metaOffset); + + return this.CompareFieldValue(leftKey, leftKeyLen, rightKey, rightKeyLen); + } + + /// + /// Sorts the array structure using the hybrid row binary + /// collation. + /// + /// The scope to be sorted. + /// A edit that points at the scope. + /// + /// A unique index array structure that identifies the row offsets of each + /// element in the scope. + /// + /// true if the array was sorted, false if a duplicate was found during sorting. + /// + /// Implementation Note: + /// This method MUST guarantee that if at least one duplicate exists it will be found. + /// Insertion Sort is used for this purpose as it guarantees that each value is eventually compared + /// against its previous item in sorted order. If any two successive items are the same they must be + /// duplicates. + /// + /// Other search algorithms, such as Quick Sort or Merge Sort, may offer fewer comparisons in the + /// limit but don't necessarily guarantee that duplicates will be discovered. If an alternative + /// algorithm is used, then an independent duplicate pass MUST be employed. + /// + /// + /// Under the current operational assumptions, the expected cardinality of sets and maps is + /// expected to be relatively small. If this assumption changes, Insertion Sort may no longer be the + /// best choice. + /// + /// + private bool InsertionSort(ref RowCursor scope, ref RowCursor dstEdit, Span uniqueIndex) + { + RowCursor leftEdit = dstEdit; + RowCursor rightEdit = dstEdit; + + for (int i = 1; i < uniqueIndex.Length; i++) + { + UniqueIndexItem x = uniqueIndex[i]; + leftEdit.cellType = LayoutType.FromCode(x.Code); + leftEdit.metaOffset = x.MetaOffset; + leftEdit.valueOffset = x.ValueOffset; + int leftBytes = x.Size - (x.ValueOffset - x.MetaOffset); + + // Walk backwards searching for the insertion point for the item as position i. + int j; + for (j = i - 1; j >= 0; j--) + { + UniqueIndexItem y = uniqueIndex[j]; + rightEdit.cellType = LayoutType.FromCode(y.Code); + rightEdit.metaOffset = y.MetaOffset; + rightEdit.valueOffset = y.ValueOffset; + + int cmp; + if (scope.scopeType is LayoutTypedMap) + { + cmp = this.CompareKeyValueFieldValue(leftEdit, rightEdit); + } + else + { + int rightBytes = y.Size - (y.ValueOffset - y.MetaOffset); + cmp = this.CompareFieldValue(leftEdit, leftBytes, rightEdit, rightBytes); + } + + // If there are duplicates then fail. + if (cmp == 0) + { + return false; + } + + if (cmp > 0) + { + break; + } + + // Swap the jth item to the right to make space for the ith item which is smaller. + uniqueIndex[j + 1] = uniqueIndex[j]; + } + + // Insert the ith item into the sorted array. + uniqueIndex[j + 1] = x; + } + + return true; + } + + internal int ReadSparsePathLen(Layout layout, int offset, out int pathLenInBytes, out int pathOffset) + { + int token = (int)this.Read7BitEncodedUInt(offset, out int sizeLenInBytes); + if (token < layout.Tokenizer.Count) + { + pathLenInBytes = sizeLenInBytes; + pathOffset = offset; + return token; + } + + int numBytes = token - layout.Tokenizer.Count; + pathLenInBytes = numBytes + sizeLenInBytes; + pathOffset = offset + sizeLenInBytes; + return token; + } + + internal Utf8Span ReadSparsePath(ref RowCursor edit) + { + if (edit.layout.Tokenizer.TryFindString((ulong)edit.pathToken, out Utf8String path)) + { + return path.Span; + } + + int numBytes = edit.pathToken - edit.layout.Tokenizer.Count; + return Utf8Span.UnsafeFromUtf8BytesNoValidation(this.buffer.Slice(edit.pathOffset, numBytes)); + } + + private void WriteSparsePath(ref RowCursor edit, int offset) + { + // Some scopes don't encode paths, therefore the cost is always zero. + if (edit.scopeType.IsIndexedScope) + { + edit.pathToken = default; + edit.pathOffset = default; + return; + } + + Contract.Assert(!edit.layout.Tokenizer.TryFindToken(edit.writePath, out StringToken _) || !edit.writePathToken.IsNull); + if (!edit.writePathToken.IsNull) + { + edit.writePathToken.Varint.CopyTo(this.buffer.Slice(offset)); + edit.pathToken = (int)edit.writePathToken.Id; + edit.pathOffset = offset; + } + else + { + // TODO: It would be better if we could avoid allocating here when the path is UTF16. + Utf8Span span = edit.writePath.ToUtf8String(); + edit.pathToken = edit.layout.Tokenizer.Count + span.Length; + int sizeLenInBytes = this.Write7BitEncodedUInt(offset, (ulong)edit.pathToken); + edit.pathOffset = offset + sizeLenInBytes; + span.Span.CopyTo(this.buffer.Slice(offset + sizeLenInBytes)); + } + } + + private Utf8Span ReadString(int offset, out int sizeLenInBytes) + { + int numBytes = (int)this.Read7BitEncodedUInt(offset, out sizeLenInBytes); + return Utf8Span.UnsafeFromUtf8BytesNoValidation(this.buffer.Slice(offset + sizeLenInBytes, numBytes)); + } + + private int WriteString(int offset, Utf8Span value) + { + int sizeLenInBytes = this.Write7BitEncodedUInt(offset, (ulong)value.Length); + value.Span.CopyTo(this.buffer.Slice(offset + sizeLenInBytes)); + return sizeLenInBytes; + } + + private ReadOnlySpan ReadBinary(int offset, out int sizeLenInBytes) + { + int numBytes = (int)this.Read7BitEncodedUInt(offset, out sizeLenInBytes); + return this.buffer.Slice(offset + sizeLenInBytes, numBytes); + } + + private int WriteBinary(int offset, ReadOnlySpan value) + { + int sizeLenInBytes = this.Write7BitEncodedUInt(offset, (ulong)value.Length); + value.CopyTo(this.buffer.Slice(offset + sizeLenInBytes)); + return sizeLenInBytes; + } + + private int WriteBinary(int offset, ReadOnlySequence value) + { + int sizeLenInBytes = this.Write7BitEncodedUInt(offset, (ulong)value.Length); + value.CopyTo(this.buffer.Slice(offset + sizeLenInBytes)); + return sizeLenInBytes; + } + + private void Ensure(int size) + { + if (this.buffer.Length < size) + { + this.buffer = this.resizer.Resize(size, this.buffer); + } + } + + private void EnsureVariable(int offset, bool isVarint, int numBytes, bool exists, out int spaceNeeded, out int shift) + { + int spaceAvailable = 0; + ulong existingValueBytes = 0; + if (exists) + { + existingValueBytes = this.Read7BitEncodedUInt(offset, out spaceAvailable); + } + + if (isVarint) + { + spaceNeeded = numBytes; + } + else + { + spaceAvailable += (int)existingValueBytes; // size already in spaceAvailable + spaceNeeded = numBytes + RowBuffer.Count7BitEncodedUInt((ulong)numBytes); + } + + shift = spaceNeeded - spaceAvailable; + if (shift > 0) + { + this.Ensure(this.length + shift); + this.buffer.Slice(offset + spaceAvailable, this.length - (offset + spaceAvailable)).CopyTo(this.buffer.Slice(offset + spaceNeeded)); + } + else if (shift < 0) + { + this.buffer.Slice(offset + spaceAvailable, this.length - (offset + spaceAvailable)).CopyTo(this.buffer.Slice(offset + spaceNeeded)); + } + } + + [Conditional("DEBUG")] + private void ReadSparsePrimitiveTypeCode(ref RowCursor edit, LayoutType code) + { + Contract.Assert(edit.exists); + + if (edit.scopeType.HasImplicitTypeCode(ref edit)) + { + if (edit.scopeType is LayoutNullable) + { + Contract.Assert(edit.scopeTypeArgs.Count == 1); + Contract.Assert(edit.index == 1); + Contract.Assert(edit.scopeTypeArgs[0].Type == code); + Contract.Assert(edit.scopeTypeArgs[0].TypeArgs.Count == 0); + } + else if (edit.scopeType.IsFixedArity) + { + Contract.Assert(edit.scopeTypeArgs.Count > edit.index); + Contract.Assert(edit.scopeTypeArgs[edit.index].Type == code); + Contract.Assert(edit.scopeTypeArgs[edit.index].TypeArgs.Count == 0); + } + else + { + Contract.Assert(edit.scopeTypeArgs.Count == 1); + Contract.Assert(edit.scopeTypeArgs[0].Type == code); + Contract.Assert(edit.scopeTypeArgs[0].TypeArgs.Count == 0); + } + } + else + { + if (code == LayoutType.Boolean) + { + code = this.ReadSparseTypeCode(edit.metaOffset); + Contract.Assert(code == LayoutType.Boolean || code == LayoutType.BooleanFalse); + } + else + { + Contract.Assert(this.ReadSparseTypeCode(edit.metaOffset) == code); + } + } + + if (edit.scopeType.IsIndexedScope) + { + Contract.Assert(edit.pathOffset == default); + Contract.Assert(edit.pathToken == default); + } + else + { + int token = this.ReadSparsePathLen(edit.layout, edit.metaOffset + sizeof(LayoutCode), out int _, out int pathOffset); + Contract.Assert(edit.pathOffset == pathOffset); + Contract.Assert(edit.pathToken == token); + } + } + + private void WriteSparseMetadata(ref RowCursor edit, LayoutType cellType, TypeArgumentList typeArgs, int metaBytes) + { + int metaOffset = edit.metaOffset; + if (!edit.scopeType.HasImplicitTypeCode(ref edit)) + { + metaOffset += cellType.WriteTypeArgument(ref this, metaOffset, typeArgs); + } + + this.WriteSparsePath(ref edit, metaOffset); + edit.valueOffset = edit.metaOffset + metaBytes; + Contract.Assert(edit.valueOffset == edit.metaOffset + metaBytes); + } + + /// + /// + /// . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureSparse( + ref RowCursor edit, + LayoutType cellType, + TypeArgumentList typeArgs, + int numBytes, + UpdateOptions options, + out int metaBytes, + out int spaceNeeded, + out int shift) + { + this.EnsureSparse(ref edit, cellType, typeArgs, numBytes, (RowOptions)options, out metaBytes, out spaceNeeded, out shift); + } + + /// Ensure that sufficient space exists in the row buffer to write the current value. + /// + /// The prepared edit indicating where and in what context the current write will + /// happen. + /// + /// The type of the field to be written. + /// The type arguments of the field to be written. + /// The number of bytes needed to encode the value of the field to be written. + /// The kind of edit to be performed. + /// + /// On success, the number of bytes needed to encode the metadata of the new + /// field. + /// + /// + /// On success, the number of bytes needed in total to encode the new field + /// and its metadata. + /// + /// + /// On success, the number of bytes the length of the row buffer was increased + /// (which may be negative if the row buffer was shrunk). + /// + private void EnsureSparse( + ref RowCursor edit, + LayoutType cellType, + TypeArgumentList typeArgs, + int numBytes, + RowOptions options, + out int metaBytes, + out int spaceNeeded, + out int shift) + { + int metaOffset = edit.metaOffset; + int spaceAvailable = 0; + + // Compute the metadata offsets + if (edit.scopeType.HasImplicitTypeCode(ref edit)) + { + metaBytes = 0; + } + else + { + metaBytes = cellType.CountTypeArgument(typeArgs); + } + + if (!edit.scopeType.IsIndexedScope) + { + Contract.Assert(edit.writePath != default(UtfAnyString)); + int pathLenInBytes = RowBuffer.CountSparsePath(ref edit); + metaBytes += pathLenInBytes; + } + + if (edit.exists) + { + // Compute value offset for existing value to be overwritten. + spaceAvailable = this.SparseComputeSize(ref edit); + } + + spaceNeeded = options == RowOptions.Delete ? 0 : metaBytes + numBytes; + shift = spaceNeeded - spaceAvailable; + if (shift > 0) + { + this.Ensure(this.length + shift); + } + + this.buffer.Slice(metaOffset + spaceAvailable, this.length - (metaOffset + spaceAvailable)) + .CopyTo(this.buffer.Slice(metaOffset + spaceNeeded)); + +#if DEBUG + if (shift < 0) + { + // Fill deleted bits (in debug builds) to detect overflow/alignment errors. + this.buffer.Slice(this.length + shift, -shift).Fill(0xFF); + } +#endif + + // Update the stored size (fixed arity scopes don't store the size because it is implied by the type args). + if (edit.scopeType.IsSizedScope && !edit.scopeType.IsFixedArity) + { + if ((options == RowOptions.Insert) || (options == RowOptions.InsertAt) || ((options == RowOptions.Upsert) && !edit.exists)) + { + // Add one to the current scope count. + Contract.Assert(!edit.exists); + this.IncrementUInt32(edit.start, 1); + edit.count++; + } + else if ((options == RowOptions.Delete) && edit.exists) + { + // Subtract one from the current scope count. + Contract.Assert(this.ReadUInt32(edit.start) > 0); + this.DecrementUInt32(edit.start, 1); + edit.count--; + } + } + + if (options == RowOptions.Delete) + { + edit.cellType = default; + edit.cellTypeArgs = default; + edit.exists = false; + } + else + { + edit.cellType = cellType; + edit.cellTypeArgs = typeArgs; + edit.exists = true; + } + } + + /// Read the metadata of an encoded sparse field. + /// The edit structure to fill in. + /// + /// + /// On success, the path of the field at the given offset, otherwise + /// undefined. + /// + /// + /// On success, the offset to the metadata of the field, otherwise a + /// location to insert the field. + /// + /// + /// On success, the layout code of the existing field, otherwise + /// undefined. + /// + /// + /// On success, the type args of the existing field, otherwise + /// undefined. + /// + /// + /// On success, the offset to the value of the field, otherwise + /// undefined. + /// . + /// + private void ReadSparseMetadata(ref RowCursor edit) + { + if (edit.scopeType.HasImplicitTypeCode(ref edit)) + { + edit.scopeType.SetImplicitTypeCode(ref edit); + edit.valueOffset = edit.metaOffset; + } + else + { + edit.cellType = this.ReadSparseTypeCode(edit.metaOffset); + edit.valueOffset = edit.metaOffset + sizeof(LayoutCode); + edit.cellTypeArgs = TypeArgumentList.Empty; + if (edit.cellType is LayoutEndScope) + { + // Reached end of current scope without finding another field. + edit.pathToken = default; + edit.pathOffset = default; + edit.valueOffset = edit.metaOffset; + return; + } + + edit.cellTypeArgs = edit.cellType.ReadTypeArgumentList(ref this, edit.valueOffset, out int sizeLenInBytes); + edit.valueOffset += sizeLenInBytes; + } + + edit.scopeType.ReadSparsePath(ref this, ref edit); + } + + /// Compute the size of a sparse field. + /// The edit structure describing the field to measure. + /// The length (in bytes) of the encoded field including the metadata and the value. + private int SparseComputeSize(ref RowCursor edit) + { + if (!(edit.cellType is LayoutScope)) + { + return this.SparseComputePrimitiveSize(edit.cellType, edit.metaOffset, edit.valueOffset); + } + + // Compute offset to end of value for current value. + RowCursor newScope = this.SparseIteratorReadScope(ref edit, immutable: true); + return this.SkipScope(ref newScope) - edit.metaOffset; + } + + /// Compute the size of a sparse (primitive) field. + /// The type of the current sparse field. + /// The 0-based offset from the beginning of the row where the field begins. + /// + /// The 0-based offset from the beginning of the row where the field's value + /// begins. + /// + /// The length (in bytes) of the encoded field including the metadata and the value. + private int SparseComputePrimitiveSize(LayoutType cellType, int metaOffset, int valueOffset) + { + // JTHTODO: convert to a virtual? + int metaBytes = valueOffset - metaOffset; + LayoutCode code = cellType.LayoutCode; + switch (code) + { + case LayoutCode.Null: + Contract.Assert(LayoutType.Null.Size == 0); + return metaBytes; + + case LayoutCode.Boolean: + case LayoutCode.BooleanFalse: + Contract.Assert(LayoutType.Boolean.Size == 0); + return metaBytes; + + case LayoutCode.Int8: + return metaBytes + LayoutType.Int8.Size; + + case LayoutCode.Int16: + return metaBytes + LayoutType.Int16.Size; + + case LayoutCode.Int32: + return metaBytes + LayoutType.Int32.Size; + + case LayoutCode.Int64: + return metaBytes + LayoutType.Int64.Size; + + case LayoutCode.UInt8: + return metaBytes + LayoutType.UInt8.Size; + + case LayoutCode.UInt16: + return metaBytes + LayoutType.UInt16.Size; + + case LayoutCode.UInt32: + return metaBytes + LayoutType.UInt32.Size; + + case LayoutCode.UInt64: + return metaBytes + LayoutType.UInt64.Size; + + case LayoutCode.Float32: + return metaBytes + LayoutType.Float32.Size; + + case LayoutCode.Float64: + return metaBytes + LayoutType.Float64.Size; + + case LayoutCode.Float128: + return metaBytes + LayoutType.Float128.Size; + + case LayoutCode.Decimal: + return metaBytes + LayoutType.Decimal.Size; + + case LayoutCode.DateTime: + return metaBytes + LayoutType.DateTime.Size; + + case LayoutCode.UnixDateTime: + return metaBytes + LayoutType.UnixDateTime.Size; + + case LayoutCode.Guid: + return metaBytes + LayoutType.Guid.Size; + + case LayoutCode.MongoDbObjectId: + return metaBytes + LayoutType.MongoDbObjectId.Size; + + case LayoutCode.Utf8: + case LayoutCode.Binary: +#pragma warning disable SA1137 // Elements should have the same indentation + { + int numBytes = (int)this.Read7BitEncodedUInt(metaOffset + metaBytes, out int sizeLenInBytes); + return metaBytes + sizeLenInBytes + numBytes; + } + + case LayoutCode.VarInt: + case LayoutCode.VarUInt: + { + this.Read7BitEncodedUInt(metaOffset + metaBytes, out int sizeLenInBytes); + return metaBytes + sizeLenInBytes; + } +#pragma warning restore SA1137 // Elements should have the same indentation + + default: + Contract.Fail($"Not Implemented: {code}"); + return 0; + } + } + + /// Return the size (in bytes) of the default sparse value for the type. + /// The type of the default value. + /// + private int CountDefaultValue(LayoutType code, TypeArgumentList typeArgs) + { + // JTHTODO: convert to a virtual? + switch (code) + { + case LayoutNull _: + case LayoutBoolean _: + return 1; + + case LayoutInt8 _: + return LayoutType.Int8.Size; + + case LayoutInt16 _: + return LayoutType.Int16.Size; + + case LayoutInt32 _: + return LayoutType.Int32.Size; + + case LayoutInt64 _: + return LayoutType.Int64.Size; + + case LayoutUInt8 _: + return LayoutType.UInt8.Size; + + case LayoutUInt16 _: + return LayoutType.UInt16.Size; + + case LayoutUInt32 _: + return LayoutType.UInt32.Size; + + case LayoutUInt64 _: + return LayoutType.UInt64.Size; + + case LayoutFloat32 _: + return LayoutType.Float32.Size; + + case LayoutFloat64 _: + return LayoutType.Float64.Size; + + case LayoutFloat128 _: + return LayoutType.Float128.Size; + + case LayoutDecimal _: + return LayoutType.Decimal.Size; + + case LayoutDateTime _: + return LayoutType.DateTime.Size; + + case LayoutUnixDateTime _: + return LayoutType.UnixDateTime.Size; + + case LayoutGuid _: + return LayoutType.Guid.Size; + + case LayoutMongoDbObjectId _: + return LayoutType.MongoDbObjectId.Size; + + case LayoutUtf8 _: + case LayoutBinary _: + case LayoutVarInt _: + case LayoutVarUInt _: + + // Variable length types preceded by their varuint size take 1 byte for a size of 0. + return 1; + + case LayoutObject _: + case LayoutArray _: + + // Variable length sparse collection scopes take 1 byte for the end-of-scope terminator. + return sizeof(LayoutCode); + + case LayoutTypedArray _: + case LayoutTypedSet _: + case LayoutTypedMap _: + + // Variable length typed collection scopes preceded by their scope size take sizeof(uint) for a size of 0. + return sizeof(uint); + + case LayoutTuple _: + + // Fixed arity sparse collections take 1 byte for end-of-scope plus a null for each element. + return sizeof(LayoutCode) + (sizeof(LayoutCode) * typeArgs.Count); + + case LayoutTypedTuple _: + case LayoutTagged _: + case LayoutTagged2 _: + + // Fixed arity typed collections take the sum of the default values of each element. The scope size is implied by the arity. + int sum = 0; + foreach (TypeArgument arg in typeArgs) + { + sum += this.CountDefaultValue(arg.Type, arg.TypeArgs); + } + + return sum; + + case LayoutNullable _: + + // Nullables take the default values of the value plus null. The scope size is implied by the arity. + return 1 + this.CountDefaultValue(typeArgs[0].Type, typeArgs[0].TypeArgs); + + case LayoutUDT _: + Layout udt = this.resolver.Resolve(typeArgs.SchemaId); + return udt.Size + sizeof(LayoutCode); + + default: + Contract.Fail($"Not Implemented: {code}"); + return 0; + } + } + + private int WriteDefaultValue(int offset, LayoutType code, TypeArgumentList typeArgs) + { + // JTHTODO: convert to a virtual? + switch (code) + { + case LayoutNull _: + this.WriteSparseTypeCode(offset, code.LayoutCode); + return 1; + + case LayoutBoolean _: + this.WriteSparseTypeCode(offset, LayoutCode.BooleanFalse); + return 1; + + case LayoutInt8 _: + this.WriteInt8(offset, 0); + return LayoutType.Int8.Size; + + case LayoutInt16 _: + this.WriteInt16(offset, 0); + return LayoutType.Int16.Size; + + case LayoutInt32 _: + this.WriteInt32(offset, 0); + return LayoutType.Int32.Size; + + case LayoutInt64 _: + this.WriteInt64(offset, 0); + return LayoutType.Int64.Size; + + case LayoutUInt8 _: + this.WriteUInt8(offset, 0); + return LayoutType.UInt8.Size; + + case LayoutUInt16 _: + this.WriteUInt16(offset, 0); + return LayoutType.UInt16.Size; + + case LayoutUInt32 _: + this.WriteUInt32(offset, 0); + return LayoutType.UInt32.Size; + + case LayoutUInt64 _: + this.WriteUInt64(offset, 0); + return LayoutType.UInt64.Size; + + case LayoutFloat32 _: + this.WriteFloat32(offset, 0); + return LayoutType.Float32.Size; + + case LayoutFloat64 _: + this.WriteFloat64(offset, 0); + return LayoutType.Float64.Size; + + case LayoutFloat128 _: + this.WriteFloat128(offset, default); + return LayoutType.Float128.Size; + + case LayoutDecimal _: + this.WriteDecimal(offset, 0); + return LayoutType.Decimal.Size; + + case LayoutDateTime _: + this.WriteDateTime(offset, default); + return LayoutType.DateTime.Size; + + case LayoutUnixDateTime _: + this.WriteUnixDateTime(offset, default); + return LayoutType.UnixDateTime.Size; + + case LayoutGuid _: + this.WriteGuid(offset, default); + return LayoutType.Guid.Size; + + case LayoutMongoDbObjectId _: + this.WriteMongoDbObjectId(offset, default); + return LayoutType.MongoDbObjectId.Size; + + case LayoutUtf8 _: + case LayoutBinary _: + case LayoutVarInt _: + case LayoutVarUInt _: + + // Variable length types preceded by their varuint size take 1 byte for a size of 0. + return this.Write7BitEncodedUInt(offset, 0); + + case LayoutObject _: + case LayoutArray _: + + // Variable length sparse collection scopes take 1 byte for the end-of-scope terminator. + this.WriteSparseTypeCode(offset, LayoutCode.EndScope); + return sizeof(LayoutCode); + + case LayoutTypedArray _: + case LayoutTypedSet _: + case LayoutTypedMap _: + + // Variable length typed collection scopes preceded by their scope size take sizeof(uint) for a size of 0. + this.WriteUInt32(offset, 0); + return sizeof(uint); + + case LayoutTuple _: + + // Fixed arity sparse collections take 1 byte for end-of-scope plus a null for each element. + for (int i = 0; i < typeArgs.Count; i++) + { + this.WriteSparseTypeCode(offset, LayoutCode.Null); + } + + this.WriteSparseTypeCode(offset, LayoutCode.EndScope); + return sizeof(LayoutCode) + (sizeof(LayoutCode) * typeArgs.Count); + + case LayoutTypedTuple _: + case LayoutTagged _: + case LayoutTagged2 _: + + // Fixed arity typed collections take the sum of the default values of each element. The scope size is implied by the arity. + int sum = 0; + foreach (TypeArgument arg in typeArgs) + { + sum += this.WriteDefaultValue(offset + sum, arg.Type, arg.TypeArgs); + } + + return sum; + + case LayoutNullable _: + + // Nullables take the default values of the value plus null. The scope size is implied by the arity. + this.WriteInt8(offset, 0); + return 1 + this.WriteDefaultValue(offset + 1, typeArgs[0].Type, typeArgs[0].TypeArgs); + + case LayoutUDT _: + + // Clear all presence bits. + Layout udt = this.resolver.Resolve(typeArgs.SchemaId); + this.buffer.Slice(offset, udt.Size).Fill(0); + + // Write scope terminator. + this.WriteSparseTypeCode(offset + udt.Size, LayoutCode.EndScope); + return udt.Size + sizeof(LayoutCode); + + default: + Contract.Fail($"Not Implemented: {code}"); + return 0; + } + } + + /// + /// represents a single item within a set/map scope that needs + /// to be indexed. + /// + /// + /// This structure is used when rebuilding a set/map index during row streaming via + /// . + /// + /// Each item encodes its offsets and length within the row. + /// + [DebuggerDisplay("{MetaOffset}/{ValueOffset}")] + private struct UniqueIndexItem + { + /// The layout code of the value. + public LayoutCode Code; + + /// + /// If existing, the offset to the metadata of the existing field, otherwise the location to + /// insert a new field. + /// + public int MetaOffset; + + /// If existing, the offset to the value of the existing field, otherwise undefined. + public int ValueOffset; + + /// Size of the target element. + public int Size; + } + } +} diff --git a/dotnet/src/HybridRow/RowCursor.cs b/dotnet/src/HybridRow/RowCursor.cs new file mode 100644 index 0000000..e9d1867 --- /dev/null +++ b/dotnet/src/HybridRow/RowCursor.cs @@ -0,0 +1,332 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1307 // Accessible fields should begin with upper-case letter + +// ReSharper disable InconsistentNaming +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System.Diagnostics; + using System.Runtime.CompilerServices; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + // ReSharper disable UseNameofExpression + [DebuggerDisplay("{ToString()}")] + public struct RowCursor + { + /// The layout describing the contents of the scope, or null if the scope is unschematized. + internal Layout layout; + + /// The kind of scope within which this edit was prepared. + internal LayoutScope scopeType; + + /// The type parameters of the scope within which this edit was prepared. + internal TypeArgumentList scopeTypeArgs; + + /// If true, this scope's nested fields cannot be updated individually. + /// The entire scope can still be replaced. + internal bool immutable; + + /// If true, this scope is an unique index scope whose index will be built after its items are written. + internal bool deferUniqueIndex; + + /// + /// The 0-based byte offset from the beginning of the row where the first sparse field within + /// the scope begins. + /// + internal int start; + + /// True if an existing field matching the search criteria was found. + internal bool exists; + + /// If existing, the scope relative path for writing. + internal UtfAnyString writePath; + + /// If WritePath is tokenized, then its token. + internal StringToken writePathToken; + + /// If existing, the offset scope relative path for reading. + internal int pathOffset; + + /// If existing, the layout string token of scope relative path for reading. + internal int pathToken; + + /// + /// If existing, the offset to the metadata of the existing field, otherwise the location to + /// insert a new field. + /// + internal int metaOffset; + + /// If existing, the layout code of the existing field, otherwise undefined. + internal LayoutType cellType; + + /// If existing, the offset to the value of the existing field, otherwise undefined. + internal int valueOffset; + + /// + /// If existing, the offset to the end of the existing field. Used as a hint when skipping + /// forward. + /// + internal int endOffset; + + /// For sized scopes (e.g. Typed Array), the number of elements. + internal int count; + + /// For indexed scopes (e.g. Array), the 0-based index into the scope of the sparse field. + internal int index; + + /// For types with generic parameters (e.g. , the type parameters. + internal TypeArgumentList cellTypeArgs; + + public static RowCursor Create(ref RowBuffer row) + { + SchemaId schemaId = row.ReadSchemaId(1); + Layout layout = row.Resolver.Resolve(schemaId); + int sparseSegmentOffset = row.ComputeVariableValueOffset(layout, HybridRowHeader.Size, layout.NumVariable); + return new RowCursor + { + layout = layout, + scopeType = LayoutType.UDT, + scopeTypeArgs = new TypeArgumentList(schemaId), + start = HybridRowHeader.Size, + metaOffset = sparseSegmentOffset, + valueOffset = sparseSegmentOffset, + }; + } + + public static ref RowCursor Create(ref RowBuffer row, out RowCursor cursor) + { + SchemaId schemaId = row.ReadSchemaId(1); + Layout layout = row.Resolver.Resolve(schemaId); + int sparseSegmentOffset = row.ComputeVariableValueOffset(layout, HybridRowHeader.Size, layout.NumVariable); + cursor = new RowCursor + { + layout = layout, + scopeType = LayoutType.UDT, + scopeTypeArgs = new TypeArgumentList(schemaId), + start = HybridRowHeader.Size, + metaOffset = sparseSegmentOffset, + valueOffset = sparseSegmentOffset, + }; + + return ref cursor; + } + + public static ref RowCursor CreateForAppend(ref RowBuffer row, out RowCursor cursor) + { + SchemaId schemaId = row.ReadSchemaId(1); + Layout layout = row.Resolver.Resolve(schemaId); + cursor = new RowCursor + { + layout = layout, + scopeType = LayoutType.UDT, + scopeTypeArgs = new TypeArgumentList(schemaId), + start = HybridRowHeader.Size, + metaOffset = row.Length, + valueOffset = row.Length, + }; + + return ref cursor; + } + + /// For indexed scopes (e.g. Array), the 0-based index into the scope of the next insertion. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public int Index + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.index; + } + + /// If true, this scope's nested fields cannot be updated individually. + /// The entire scope can still be replaced. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public bool Immutable + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.immutable; + } + + /// The kind of scope. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public LayoutType ScopeType + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.scopeType; + } + + /// For types with generic parameters (e.g. , the type parameters. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public TypeArgumentList ScopeTypeArgs + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.scopeTypeArgs; + } + + /// The layout describing the contents of the scope, or null if the scope is unschematized. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public Layout Layout + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => this.layout; + } + + /// The full logical type. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public TypeArgument TypeArg + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => new TypeArgument(this.cellType, this.cellTypeArgs); + } + + public override string ToString() + { + try + { + if (this.scopeType == null) + { + return ""; + } + + TypeArgument scopeTypeArg = (this.scopeType == null) || (this.scopeType is LayoutEndScope) + ? default + : new TypeArgument(this.scopeType, this.scopeTypeArgs); + + TypeArgument typeArg = (this.cellType == null) || (this.cellType is LayoutEndScope) + ? default + : new TypeArgument(this.cellType, this.cellTypeArgs); + + string pathOrIndex = !this.writePath.IsNull ? this.writePath.ToString() : this.index.ToString(); + return $"{scopeTypeArg}[{pathOrIndex}] : {typeArg}@{this.metaOffset}/{this.valueOffset}" + + (this.immutable ? " immutable" : string.Empty); + } + catch + { + return ""; + } + } + } + + public static class RowCursorExtensions + { + /// Makes a copy of the current cursor. + /// + /// The two cursors will have independent and unconnected lifetimes after cloning. However, + /// mutations to a can invalidate any active cursors over the same row. + /// + public static ref RowCursor Clone(this in RowCursor src, out RowCursor dest) + { + dest = src; + return ref dest; + } + + /// Returns an equivalent scope that is read-only. + public static ref RowCursor AsReadOnly(this in RowCursor src, out RowCursor dest) + { + dest = src; + dest.immutable = true; + return ref dest; + } + + public static ref RowCursor Find(this ref RowCursor edit, ref RowBuffer row, UtfAnyString path) + { + Contract.Requires(!edit.scopeType.IsIndexedScope); + + if (!(edit.cellType is LayoutEndScope)) + { + while (row.SparseIteratorMoveNext(ref edit)) + { + if (path.Equals(row.ReadSparsePath(ref edit))) + { + edit.exists = true; + break; + } + } + } + + edit.writePath = path; + edit.writePathToken = default; + return ref edit; + } + + public static ref RowCursor Find(this ref RowCursor edit, ref RowBuffer row, in StringToken pathToken) + { + Contract.Requires(!edit.scopeType.IsIndexedScope); + + if (!(edit.cellType is LayoutEndScope)) + { + while (row.SparseIteratorMoveNext(ref edit)) + { + if (pathToken.Id == (ulong)edit.pathToken) + { + edit.exists = true; + break; + } + } + } + + edit.writePath = pathToken.Path; + edit.writePathToken = pathToken; + return ref edit; + } + + public static bool MoveNext(this ref RowCursor edit, ref RowBuffer row) + { + edit.writePath = default; + edit.writePathToken = default; + return row.SparseIteratorMoveNext(ref edit); + } + + public static bool MoveTo(this ref RowCursor edit, ref RowBuffer row, int index) + { + Contract.Assert(edit.index <= index); + edit.writePath = default; + edit.writePathToken = default; + while (edit.index < index) + { + if (!row.SparseIteratorMoveNext(ref edit)) + { + return false; + } + } + + return true; + } + + public static bool MoveNext(this ref RowCursor edit, ref RowBuffer row, ref RowCursor childScope) + { + if (childScope.scopeType != null) + { + edit.Skip(ref row, ref childScope); + } + + return edit.MoveNext(ref row); + } + + public static void Skip(this ref RowCursor edit, ref RowBuffer row, ref RowCursor childScope) + { + Contract.Requires(childScope.start == edit.valueOffset); + if (!(childScope.cellType is LayoutEndScope)) + { + while (row.SparseIteratorMoveNext(ref childScope)) + { + } + } + + if (childScope.scopeType.IsSizedScope) + { + edit.endOffset = childScope.metaOffset; + } + else + { + edit.endOffset = childScope.metaOffset + sizeof(LayoutCode); // Move past the end of scope marker. + } + +#if DEBUG + childScope = default; +#endif + } + } +} diff --git a/dotnet/src/HybridRow/RowOptions.cs b/dotnet/src/HybridRow/RowOptions.cs new file mode 100644 index 0000000..5bff003 --- /dev/null +++ b/dotnet/src/HybridRow/RowOptions.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + /// Describes the desired behavior when mutating a hybrid row. + internal enum RowOptions + { + None = 0, + + /// Overwrite an existing value. + /// + /// An existing value is assumed to exist at the offset provided. The existing value is + /// replaced inline. The remainder of the row is resized to accomodate either an increase or decrease + /// in required space. + /// + Update = 1, + + /// Insert a new value. + /// + /// An existing value is assumed NOT to exist at the offset provided. The new value is + /// inserted immediately at the offset. The remainder of the row is resized to accomodate either an + /// increase or decrease in required space. + /// + Insert = 2, + + /// Update an existing value or insert a new value, if no value exists. + /// + /// If a value exists, then this operation becomes , otherwise it + /// becomes . + /// + Upsert = 3, + + /// Insert a new value moving existing values to the right. + /// + /// Within an array scope, inserts a new value immediately at the index moving all subsequent + /// items to the right. In any other scope behaves the same as . + /// + InsertAt = 4, + + /// Delete an existing value. + /// + /// If a value exists, then it is removed. The remainder of the row is resized to accomodate + /// a decrease in required space. If no value exists this operation is a no-op. + /// + Delete = 5, + } +} diff --git a/dotnet/src/HybridRow/SchemaId.cs b/dotnet/src/HybridRow/SchemaId.cs new file mode 100644 index 0000000..6a446a6 --- /dev/null +++ b/dotnet/src/HybridRow/SchemaId.cs @@ -0,0 +1,98 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + using System.Diagnostics; + using System.Runtime.InteropServices; + using Microsoft.Azure.Cosmos.Core; + using Newtonsoft.Json; + + /// The unique identifier for a schema. + /// Identifiers must be unique within the scope of the database in which they are used. + [JsonConverter(typeof(SchemaIdConverter))] + [DebuggerDisplay("{" + nameof(SchemaId.Id) + "}")] + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public readonly struct SchemaId : IEquatable + { + public const int Size = sizeof(int); + public static readonly SchemaId Invalid = default; + + /// Initializes a new instance of the struct. + /// The underlying globally unique identifier of the schema. + public SchemaId(int id) + { + this.Id = id; + } + + /// The underlying identifier. + public int Id { get; } + + /// Operator == overload. + public static bool operator ==(SchemaId left, SchemaId right) + { + return left.Equals(right); + } + + /// Operator != overload. + public static bool operator !=(SchemaId left, SchemaId right) + { + return !left.Equals(right); + } + + /// overload. + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + return obj is SchemaId && this.Equals((SchemaId)obj); + } + + /// overload. + public override int GetHashCode() + { + return this.Id.GetHashCode(); + } + + /// Returns true if this is the same as . + /// The value to compare against. + /// True if the two values are the same. + public bool Equals(SchemaId other) + { + return this.Id == other.Id; + } + + /// overload. + public override string ToString() + { + return this.Id.ToString(); + } + + /// Helper class for parsing from JSON. + internal class SchemaIdConverter : JsonConverter + { + public override bool CanWrite => true; + + public override bool CanConvert(Type objectType) + { + return typeof(SchemaId).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + Contract.Requires(reader.TokenType == JsonToken.Integer); + return new SchemaId(checked((int)(long)reader.Value)); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + writer.WriteValue((long)((SchemaId)value).Id); + } + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/ArrayPropertyType.cs b/dotnet/src/HybridRow/Schemas/ArrayPropertyType.cs new file mode 100644 index 0000000..e5f8c1f --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/ArrayPropertyType.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// Array properties represent an unbounded set of zero or more items. + /// + /// Arrays may be typed or untyped. Within typed arrays, all items MUST be the same type. The + /// type of items is specified via . Typed arrays may be stored more efficiently + /// than untyped arrays. When is unspecified, the array is untyped and its items + /// may be heterogeneous. + /// + public class ArrayPropertyType : ScopePropertyType + { + /// (Optional) type of the elements of the array, if a typed array, otherwise null. + [JsonProperty(PropertyName = "items")] + public PropertyType Items { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/MapPropertyType.cs b/dotnet/src/HybridRow/Schemas/MapPropertyType.cs new file mode 100644 index 0000000..393b3b8 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/MapPropertyType.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// + /// Map properties represent an unbounded set of zero or more key-value pairs with unique + /// keys. + /// + /// + /// Maps are typed or untyped. Within typed maps, all key MUST be the same type, and all + /// values MUST be the same type. The type of both key and values is specified via + /// and respectively. Typed maps may be stored more efficiently than untyped + /// maps. When or is unspecified or marked + /// , the map is untyped and its key and/or values may be heterogeneous. + /// + public class MapPropertyType : ScopePropertyType + { + /// (Optional) type of the keys of the map, if a typed map, otherwise null. + [JsonProperty(PropertyName = "keys")] + public PropertyType Keys { get; set; } + + /// (Optional) type of the values of the map, if a typed map, otherwise null. + [JsonProperty(PropertyName = "values")] + public PropertyType Values { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/Namespace.cs b/dotnet/src/HybridRow/Schemas/Namespace.cs new file mode 100644 index 0000000..e46ab34 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/Namespace.cs @@ -0,0 +1,65 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1716 // Identifiers should not match keywords + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Collections.Generic; + using Newtonsoft.Json; + + [JsonObject] + public class Namespace + { + /// + /// The standard settings used by the JSON parser for interpreting + /// documents. + /// + private static readonly JsonSerializerSettings NamespaceParseSettings = new JsonSerializerSettings() + { + CheckAdditionalContent = true, + }; + + /// The set of schemas that make up the . + private List schemas; + + /// Initializes a new instance of the class. + public Namespace() + { + this.Schemas = new List(); + } + + /// The version of the HybridRow Schema Definition Language used to encode this namespace. + [JsonProperty(PropertyName = "version")] + public SchemaLanguageVersion Version { get; set; } + + /// The fully qualified identifier of the namespace. + [JsonProperty(PropertyName = "name")] + public string Name { get; set; } + + /// The set of schemas that make up the . + /// + /// Namespaces may consist of zero or more table schemas along with zero or more UDT schemas. + /// Table schemas can only reference UDT schemas defined in the same namespace. UDT schemas can + /// contain nested UDTs whose schemas are defined within the same namespace. + /// + [JsonProperty(PropertyName = "schemas")] + public List Schemas + { + get => this.schemas; + + set => this.schemas = value ?? new List(); + } + + /// Parse a JSON document and return a full namespace. + /// The JSON text to parse. + /// A namespace containing a set of logical schemas. + public static Namespace Parse(string json) + { + Namespace ns = JsonConvert.DeserializeObject(json, Namespace.NamespaceParseSettings); + SchemaValidator.Validate(ns); + return ns; + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/ObjectPropertyType.cs b/dotnet/src/HybridRow/Schemas/ObjectPropertyType.cs new file mode 100644 index 0000000..2ec8778 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/ObjectPropertyType.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Collections.Generic; + using Newtonsoft.Json; + + /// Object properties represent nested structures. + /// + /// Object properties map to multiple columns depending on the number of internal properties + /// within the defined object structure. Object properties are provided as a convince in schema + /// design. They are effectively equivalent to defining the same properties explicitly via + /// with nested property paths. + /// + public class ObjectPropertyType : ScopePropertyType + { + /// A list of zero or more property definitions that define the columns within the schema. + private List properties; + + /// Initializes a new instance of the class. + public ObjectPropertyType() + { + this.properties = new List(); + } + + /// A list of zero or more property definitions that define the columns within the schema. + [JsonProperty(PropertyName = "properties")] + public List Properties + { + get + { + return this.properties; + } + + set + { + this.properties = value ?? new List(); + } + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/PartitionKey.cs b/dotnet/src/HybridRow/Schemas/PartitionKey.cs new file mode 100644 index 0000000..68cf9b2 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/PartitionKey.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// Describes a property or set of properties used to partition the data set across machines. + public class PartitionKey + { + /// The logical path of the referenced property. + /// Partition keys MUST refer to properties defined within the same . + [JsonProperty(PropertyName = "path", Required = Required.Always)] + public string Path { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/PrimarySortKey.cs b/dotnet/src/HybridRow/Schemas/PrimarySortKey.cs new file mode 100644 index 0000000..9d02071 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/PrimarySortKey.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// + /// Describes a property or set of properties used to order the data set within a single + /// partition. + /// + public class PrimarySortKey + { + /// The logical path of the referenced property. + /// Primary keys MUST refer to properties defined within the same . + [JsonProperty(PropertyName = "path", Required = Required.Always)] + public string Path { get; set; } + + /// The logical path of the referenced property. + /// Primary keys MUST refer to properties defined within the same . + [JsonProperty(PropertyName = "direction", Required = Required.DisallowNull)] + public SortDirection Direction { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/PrimitivePropertyType.cs b/dotnet/src/HybridRow/Schemas/PrimitivePropertyType.cs new file mode 100644 index 0000000..9867bf7 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/PrimitivePropertyType.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// A primitive property. + /// + /// Primitive properties map to columns one-to-one. Primitive properties indicate how the + /// column should be represented within the row. + /// + public class PrimitivePropertyType : PropertyType + { + /// The maximum allowable length in bytes. + /// + /// This annotation is only valid for non-fixed length types. A value of 0 means the maximum + /// allowable length. + /// + [JsonProperty(PropertyName = "length")] + [JsonConverter(typeof(StrictIntegerConverter))] + public int Length { get; set; } + + /// Storage requirements of the property. + [JsonProperty(PropertyName = "storage")] + public StorageKind Storage { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/Property.cs b/dotnet/src/HybridRow/Schemas/Property.cs new file mode 100644 index 0000000..4a19772 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/Property.cs @@ -0,0 +1,37 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1716 // Identifiers should not match keywords + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// Describes a single property definition. + public class Property + { + /// An (optional) comment describing the purpose of this property. + /// Comments are for documentary purpose only and do not affect the property at runtime. + [JsonProperty(PropertyName = "comment", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string Comment { get; set; } + + /// The logical path of this property. + /// + /// For complex properties (e.g. objects) the logical path forms a prefix to relative paths of + /// properties defined within nested structures. + /// + /// See the logical path specification for full details on both relative and absolute paths. + /// + [JsonProperty(PropertyName = "path", Required = Required.Always)] + public string Path { get; set; } + + /// The type of the property. + /// + /// Types may be simple (e.g. int8) or complex (e.g. object). Simple types always define a + /// single column. Complex types may define one or more columns depending on their structure. + /// + [JsonProperty(PropertyName = "type", Required = Required.Always)] + public PropertyType PropertyType { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/PropertySchemaConverter.cs b/dotnet/src/HybridRow/Schemas/PropertySchemaConverter.cs new file mode 100644 index 0000000..4ba0b85 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/PropertySchemaConverter.cs @@ -0,0 +1,82 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System; + using System.Diagnostics.CodeAnalysis; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + using Newtonsoft.Json.Linq; + + /// Helper class for parsing the polymorphic subclasses from JSON. + internal class PropertySchemaConverter : JsonConverter + { + public override bool CanWrite => false; + + public override bool CanConvert(Type objectType) + { + return typeof(PropertyType).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + PropertyType p; + if (reader.TokenType != JsonToken.StartObject) + { + throw new JsonSerializationException(); + } + + JObject propSchema = JObject.Load(reader); + TypeKind propType; + + if (!propSchema.TryGetValue("type", out JToken value)) + { + throw new JsonSerializationException("Required \"type\" property missing."); + } + + using (JsonReader typeReader = value.CreateReader()) + { + typeReader.Read(); // Move to the start token + propType = (TypeKind)new StringEnumConverter(true).ReadJson(typeReader, typeof(TypeKind), null, serializer); + } + + switch (propType) + { + case TypeKind.Array: + p = new ArrayPropertyType(); + break; + case TypeKind.Set: + p = new SetPropertyType(); + break; + case TypeKind.Map: + p = new MapPropertyType(); + break; + case TypeKind.Object: + p = new ObjectPropertyType(); + break; + case TypeKind.Tuple: + p = new TuplePropertyType(); + break; + case TypeKind.Tagged: + p = new TaggedPropertyType(); + break; + case TypeKind.Schema: + p = new UdtPropertyType(); + break; + default: + p = new PrimitivePropertyType(); + break; + } + + serializer.Populate(propSchema.CreateReader(), p); + return p; + } + + [ExcludeFromCodeCoverage] + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/PropertyType.cs b/dotnet/src/HybridRow/Schemas/PropertyType.cs new file mode 100644 index 0000000..273eb43 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/PropertyType.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.ComponentModel; + using Newtonsoft.Json; + + /// The base class for property types both primitive and complex. + [JsonConverter(typeof(PropertySchemaConverter))] + public abstract class PropertyType + { + protected PropertyType() + { + this.Nullable = true; + } + + /// Api-specific type annotations for the property. + [JsonProperty(PropertyName = "apitype")] + public string ApiType { get; set; } + + /// The logical type of the property. + [JsonProperty(PropertyName = "type")] + public TypeKind Type { get; set; } + + /// True if the property can be null. + /// Default: true. + [DefaultValue(true)] + [JsonProperty(PropertyName = "nullable", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + [JsonConverter(typeof(StrictBooleanConverter))] + public bool Nullable { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/Schema.cs b/dotnet/src/HybridRow/Schemas/Schema.cs new file mode 100644 index 0000000..284afe2 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/Schema.cs @@ -0,0 +1,171 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Collections.Generic; + using System.ComponentModel; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Newtonsoft.Json; + + /// A schema describes either table or UDT metadata. + /// + /// The schema of a table or UDT describes the structure of row (i.e. which columns and the + /// types of those columns). A table schema represents the description of the contents of a collection + /// level row directly. UDTs described nested structured objects that may appear either within a table + /// column or within another UDT (i.e. nested UDTs). + /// + public class Schema + { + /// An (optional) list of zero or more logical paths that form the partition key. + private List partitionKeys; + + /// An (optional) list of zero or more logical paths that form the primary sort key. + private List primaryKeys; + + /// An (optional) list of zero or more logical paths that hold data shared by all documents that have the same partition key. + private List staticKeys; + + /// A list of zero or more property definitions that define the columns within the schema. + private List properties; + + /// Initializes a new instance of the class. + public Schema() + { + this.Type = TypeKind.Schema; + this.properties = new List(); + this.partitionKeys = new List(); + this.primaryKeys = new List(); + this.staticKeys = new List(); + } + + /// The version of the HybridRow Schema Definition Language used to encode this schema. + [JsonProperty(PropertyName = "version")] + public SchemaLanguageVersion Version { get; set; } + + /// An (optional) comment describing the purpose of this schema. + /// Comments are for documentary purpose only and do not affect the schema at runtime. + [JsonProperty(PropertyName = "comment", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string Comment { get; set; } + + /// The name of the schema. + /// + /// The name of a schema MUST be unique within its namespace. + /// + /// Names must begin with an alpha-numeric character and can only contain alpha-numeric characters and + /// underscores. + /// + [JsonProperty(PropertyName = "name", Required = Required.Always)] + public string Name { get; set; } + + /// The unique identifier for a schema. + /// Identifiers must be unique within the scope of the database in which they are used. + [JsonProperty(PropertyName = "id", Required = Required.Always)] + public SchemaId SchemaId { get; set; } + + /// Schema-wide operations. + [JsonProperty(PropertyName = "options")] + public SchemaOptions Options { get; set; } + + /// The type of this schema. This value MUST be . + [DefaultValue(TypeKind.Schema)] + [JsonProperty(PropertyName = "type", Required = Required.DisallowNull, DefaultValueHandling = DefaultValueHandling.Populate)] + public TypeKind Type { get; set; } + + /// A list of zero or more property definitions that define the columns within the schema. + /// This field is never null. + [JsonProperty(PropertyName = "properties")] + public List Properties + { + get + { + return this.properties; + } + + set + { + this.properties = value ?? new List(); + } + } + + /// An (optional) list of zero or more logical paths that form the partition key. + /// All paths referenced MUST map to a property within the schema. + /// + /// This field is never null. + [JsonProperty(PropertyName = "partitionkeys")] + public List PartitionKeys + { + get + { + return this.partitionKeys; + } + + set + { + this.partitionKeys = value ?? new List(); + } + } + + /// An (optional) list of zero or more logical paths that form the primary sort key. + /// All paths referenced MUST map to a property within the schema. + /// + /// This field is never null. + [JsonProperty(PropertyName = "primarykeys")] + public List PrimarySortKeys + { + get + { + return this.primaryKeys; + } + + set + { + this.primaryKeys = value ?? new List(); + } + } + + /// An (optional) list of zero or more logical paths that hold data shared by all documents with same partition key. + /// All paths referenced MUST map to a property within the schema. + /// + /// This field is never null. + [JsonProperty(PropertyName = "statickeys", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public List StaticKeys + { + get + { + return this.staticKeys; + } + + set + { + this.staticKeys = value ?? new List(); + } + } + + /// Parse a JSON fragment and return a schema. + /// The JSON text to parse. + /// A logical schema. + public static Schema Parse(string json) + { + return JsonConvert.DeserializeObject(json); + + // TODO: perform structural validation on the Schema after JSON parsing. + } + + /// + /// Compiles this logical schema into a physical layout that can be used to read and write + /// rows. + /// + /// The namespace within which this schema is defined. + /// The layout for the schema. + public Layout Compile(Namespace ns) + { + Contract.Requires(ns != null); + Contract.Requires(ns.Schemas.Contains(this)); + + return LayoutCompiler.Compile(ns, this); + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/SchemaException.cs b/dotnet/src/HybridRow/Schemas/SchemaException.cs new file mode 100644 index 0000000..78c75aa --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SchemaException.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.Serialization; + + [Serializable] + [ExcludeFromCodeCoverage] + public sealed class SchemaException : Exception + { + public SchemaException() + { + } + + public SchemaException(string message) + : base(message) + { + } + + public SchemaException(string message, Exception innerException) + : base(message, innerException) + { + } + + private SchemaException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/SchemaHash.cs b/dotnet/src/HybridRow/Schemas/SchemaHash.cs new file mode 100644 index 0000000..e9488e4 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SchemaHash.cs @@ -0,0 +1,222 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Internal; + + public static class SchemaHash + { + /// Computes the logical hash for a logical schema. + /// The namespace within which is defined. + /// The logical schema to compute the hash of. + /// The seed to initialized the hash function. + /// The logical 128-bit hash as a two-tuple (low, high). + public static (ulong low, ulong high) ComputeHash(Namespace ns, Schema schema, (ulong low, ulong high) seed = default) + { + (ulong low, ulong high) hash = seed; + hash = MurmurHash3.Hash128(schema.SchemaId, hash); + hash = MurmurHash3.Hash128(schema.Type, hash); + hash = SchemaHash.ComputeHash(ns, schema.Options, hash); + if (schema.PartitionKeys != null) + { + foreach (PartitionKey p in schema.PartitionKeys) + { + hash = SchemaHash.ComputeHash(ns, p, hash); + } + } + + if (schema.PrimarySortKeys != null) + { + foreach (PrimarySortKey p in schema.PrimarySortKeys) + { + hash = SchemaHash.ComputeHash(ns, p, hash); + } + } + + if (schema.StaticKeys != null) + { + foreach (StaticKey p in schema.StaticKeys) + { + hash = SchemaHash.ComputeHash(ns, p, hash); + } + } + + if (schema.Properties != null) + { + foreach (Property p in schema.Properties) + { + hash = SchemaHash.ComputeHash(ns, p, hash); + } + } + + return hash; + } + + private static (ulong low, ulong high) ComputeHash(Namespace ns, SchemaOptions options, (ulong low, ulong high) seed = default) + { + (ulong low, ulong high) hash = seed; + hash = MurmurHash3.Hash128(options?.DisallowUnschematized ?? false, hash); + hash = MurmurHash3.Hash128(options?.EnablePropertyLevelTimestamp ?? false, hash); + if (options?.DisableSystemPrefix ?? false) + { + hash = MurmurHash3.Hash128(true, hash); + } + + return hash; + } + + private static (ulong low, ulong high) ComputeHash(Namespace ns, Property p, (ulong low, ulong high) seed = default) + { + Contract.Requires(p != null); + (ulong low, ulong high) hash = seed; + hash = MurmurHash3.Hash128(p.Path, hash); + hash = SchemaHash.ComputeHash(ns, p.PropertyType, hash); + return hash; + } + + private static (ulong low, ulong high) ComputeHash(Namespace ns, PropertyType p, (ulong low, ulong high) seed = default) + { + Contract.Requires(p != null); + (ulong low, ulong high) hash = seed; + hash = MurmurHash3.Hash128(p.Type, hash); + hash = MurmurHash3.Hash128(p.Nullable, hash); + if (p.ApiType != null) + { + hash = MurmurHash3.Hash128(p.ApiType, hash); + } + + switch (p) + { + case PrimitivePropertyType pp: + hash = MurmurHash3.Hash128(pp.Storage, hash); + hash = MurmurHash3.Hash128(pp.Length, hash); + break; + case ScopePropertyType pp: + hash = MurmurHash3.Hash128(pp.Immutable, hash); + switch (p) + { + case ArrayPropertyType spp: + if (spp.Items != null) + { + hash = SchemaHash.ComputeHash(ns, spp.Items, hash); + } + + break; + case ObjectPropertyType spp: + if (spp.Properties != null) + { + foreach (Property opp in spp.Properties) + { + hash = SchemaHash.ComputeHash(ns, opp, hash); + } + } + + break; + case MapPropertyType spp: + if (spp.Keys != null) + { + hash = SchemaHash.ComputeHash(ns, spp.Keys, hash); + } + + if (spp.Values != null) + { + hash = SchemaHash.ComputeHash(ns, spp.Values, hash); + } + + break; + case SetPropertyType spp: + if (spp.Items != null) + { + hash = SchemaHash.ComputeHash(ns, spp.Items, hash); + } + + break; + case TaggedPropertyType spp: + if (spp.Items != null) + { + foreach (PropertyType pt in spp.Items) + { + hash = SchemaHash.ComputeHash(ns, pt, hash); + } + } + + break; + case TuplePropertyType spp: + if (spp.Items != null) + { + foreach (PropertyType pt in spp.Items) + { + hash = SchemaHash.ComputeHash(ns, pt, hash); + } + } + + break; + case UdtPropertyType spp: + Schema udtSchema; + if (spp.SchemaId == SchemaId.Invalid) + { + udtSchema = ns.Schemas.Find(s => s.Name == spp.Name); + } + else + { + udtSchema = ns.Schemas.Find(s => s.SchemaId == spp.SchemaId); + if (udtSchema.Name != spp.Name) + { + throw new Exception($"Ambiguous schema reference: '{spp.Name}:{spp.SchemaId}'"); + } + } + + if (udtSchema == null) + { + throw new Exception($"Cannot resolve schema reference '{spp.Name}:{spp.SchemaId}'"); + } + + hash = SchemaHash.ComputeHash(ns, udtSchema, hash); + break; + } + + break; + } + + return hash; + } + + private static (ulong low, ulong high) ComputeHash(Namespace ns, PartitionKey key, (ulong low, ulong high) seed = default) + { + (ulong low, ulong high) hash = seed; + if (key != null) + { + hash = MurmurHash3.Hash128(key.Path, hash); + } + + return hash; + } + + private static (ulong low, ulong high) ComputeHash(Namespace ns, PrimarySortKey key, (ulong low, ulong high) seed = default) + { + (ulong low, ulong high) hash = seed; + if (key != null) + { + hash = MurmurHash3.Hash128(key.Path, hash); + hash = MurmurHash3.Hash128(key.Direction, hash); + } + + return hash; + } + + private static (ulong low, ulong high) ComputeHash(Namespace ns, StaticKey key, (ulong low, ulong high) seed = default) + { + (ulong low, ulong high) hash = seed; + if (key != null) + { + hash = MurmurHash3.Hash128(key.Path, hash); + } + + return hash; + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/SchemaLanguageVersion.cs b/dotnet/src/HybridRow/Schemas/SchemaLanguageVersion.cs new file mode 100644 index 0000000..2af920e --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SchemaLanguageVersion.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1028 // Enum Storage should be Int32 + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + /// Versions of the HybridRow Schema Description Language. + [JsonConverter(typeof(StringEnumConverter), true)] // camelCase:true + public enum SchemaLanguageVersion : byte + { + /// Initial version of the HybridRow Schema Description Lanauge. + V1 = 0, + } +} diff --git a/dotnet/src/HybridRow/Schemas/SchemaOptions.cs b/dotnet/src/HybridRow/Schemas/SchemaOptions.cs new file mode 100644 index 0000000..22dc698 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SchemaOptions.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// Describes the set of options that apply to the entire schema and the way it is validated. + public class SchemaOptions + { + /// If true then structural schema validation is enabled. + /// + /// When structural schema validation is enabled then attempting to store an unschematized + /// path in the row, or a value whose type does not conform to the type constraints defined for that + /// path within the schema will lead to a schema validation error. When structural schema validation is + /// NOT enabled, then storing an unschematized path or non-confirming value will lead to a sparse + /// column override of the path. The value will be stored (and any existing value at that path will be + /// overwritten). No error will be given. + /// + [JsonProperty(PropertyName = "disallowUnschematized")] + [JsonConverter(typeof(StrictBooleanConverter))] + public bool DisallowUnschematized { get; set; } + + /// + /// If set and has the value true, then triggers behavior in the Schema that acts based on property + /// level timestamps. In Cassandra, this means that new columns are added for each top level property + /// that has values of the client side timestamp. This is then used in conflict resolution to independently + /// resolve each property based on the timestamp value of that property. + /// + [JsonProperty(PropertyName = "enablePropertyLevelTimestamp")] + [JsonConverter(typeof(StrictBooleanConverter))] + public bool EnablePropertyLevelTimestamp { get; set; } + + /// + /// If the is value true, then disables prefixing the system properties with a prefix __sys_ + /// for reserved properties owned by the store layer. + /// + [JsonProperty(PropertyName = "disableSystemPrefix", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + [JsonConverter(typeof(StrictBooleanConverter))] + public bool DisableSystemPrefix { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/SchemaValidator.cs b/dotnet/src/HybridRow/Schemas/SchemaValidator.cs new file mode 100644 index 0000000..3d1f309 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SchemaValidator.cs @@ -0,0 +1,251 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + + public static class SchemaValidator + { + public static void Validate(Namespace ns) + { + Dictionary nameVersioningCheck = new Dictionary(ns.Schemas.Count); + Dictionary<(string, SchemaId), Schema> nameDupCheck = new Dictionary<(string, SchemaId), Schema>(ns.Schemas.Count); + Dictionary idDupCheck = new Dictionary(ns.Schemas.Count); + foreach (Schema s in ns.Schemas) + { + ValidateAssert.IsValidSchemaId(s.SchemaId, "Schema id"); + ValidateAssert.IsValidIdentifier(s.Name, "Schema name"); + ValidateAssert.DuplicateCheck(s.SchemaId, s, idDupCheck, "Schema id", "Namespace"); + ValidateAssert.DuplicateCheck((s.Name, s.SchemaId), s, nameDupCheck, "Schema reference", "Namespace"); + + // Count the versions of each schema by name. + nameVersioningCheck.TryGetValue(s.Name, out int count); + nameVersioningCheck[s.Name] = count + 1; + } + + // Enable id-less Schema references for all types with a unique version in the namespace. + foreach (Schema s in ns.Schemas) + { + if (nameVersioningCheck[s.Name] == 1) + { + ValidateAssert.DuplicateCheck((s.Name, SchemaId.Invalid), s, nameDupCheck, "Schema reference", "Namespace"); + } + } + + SchemaValidator.Visit(ns, nameDupCheck, idDupCheck); + } + + /// Visit an entire namespace and validate its constraints. + /// The to validate. + /// A map from schema names within the namespace to their schemas. + /// A map from schema ids within the namespace to their schemas. + private static void Visit(Namespace ns, Dictionary<(string, SchemaId), Schema> schemas, Dictionary ids) + { + foreach (Schema s in ns.Schemas) + { + SchemaValidator.Visit(s, schemas, ids); + } + } + + /// Visit a single schema and validate its constraints. + /// The to validate. + /// A map from schema names within the namespace to their schemas. + /// A map from schema ids within the namespace to their schemas. + private static void Visit(Schema s, Dictionary<(string, SchemaId), Schema> schemas, Dictionary ids) + { + ValidateAssert.AreEqual(s.Type, TypeKind.Schema, $"The type of a schema MUST be {TypeKind.Schema}: {s.Type}"); + Dictionary pathDupCheck = new Dictionary(s.Properties.Count); + foreach (Property p in s.Properties) + { + ValidateAssert.DuplicateCheck(p.Path, p, pathDupCheck, "Property path", "Schema"); + } + + foreach (PartitionKey pk in s.PartitionKeys) + { + ValidateAssert.Exists(pk.Path, pathDupCheck, "Partition key column", "Schema"); + } + + foreach (PrimarySortKey ps in s.PrimarySortKeys) + { + ValidateAssert.Exists(ps.Path, pathDupCheck, "Primary sort key column", "Schema"); + } + + foreach (StaticKey sk in s.StaticKeys) + { + ValidateAssert.Exists(sk.Path, pathDupCheck, "Static key column", "Schema"); + } + + foreach (Property p in s.Properties) + { + SchemaValidator.Visit(p, s, schemas, ids); + } + } + + private static void Visit(Property p, Schema s, Dictionary<(string, SchemaId), Schema> schemas, Dictionary ids) + { + ValidateAssert.IsValidIdentifier(p.Path, "Property path"); + SchemaValidator.Visit(p.PropertyType, null, schemas, ids); + } + + private static void Visit( + PropertyType p, + PropertyType parent, + Dictionary<(string, SchemaId), Schema> schemas, + Dictionary ids) + { + switch (p) + { + case PrimitivePropertyType pp: + ValidateAssert.IsTrue(pp.Length >= 0, "Length MUST be positive"); + if (parent != null) + { + ValidateAssert.AreEqual(pp.Storage, StorageKind.Sparse, $"Nested fields MUST have storage {StorageKind.Sparse}"); + } + + break; + case ArrayPropertyType ap: + if (ap.Items != null) + { + SchemaValidator.Visit(ap.Items, p, schemas, ids); + } + + break; + case MapPropertyType mp: + SchemaValidator.Visit(mp.Keys, p, schemas, ids); + SchemaValidator.Visit(mp.Values, p, schemas, ids); + break; + case SetPropertyType sp: + SchemaValidator.Visit(sp.Items, p, schemas, ids); + break; + case TaggedPropertyType gp: + foreach (PropertyType item in gp.Items) + { + SchemaValidator.Visit(item, p, schemas, ids); + } + + break; + case TuplePropertyType tp: + foreach (PropertyType item in tp.Items) + { + SchemaValidator.Visit(item, p, schemas, ids); + } + + break; + case ObjectPropertyType op: + Dictionary pathDupCheck = new Dictionary(op.Properties.Count); + foreach (Property nested in op.Properties) + { + ValidateAssert.DuplicateCheck(nested.Path, nested, pathDupCheck, "Property path", "Object"); + SchemaValidator.Visit(nested.PropertyType, p, schemas, ids); + } + + break; + case UdtPropertyType up: + ValidateAssert.Exists((up.Name, up.SchemaId), schemas, "Schema reference", "Namespace"); + if (up.SchemaId != SchemaId.Invalid) + { + Schema s = ValidateAssert.Exists(up.SchemaId, ids, "Schema id", "Namespace"); + ValidateAssert.AreEqual( + up.Name, + s.Name, + $"Schema name '{up.Name}' does not match the name of schema with id '{up.SchemaId}': {s.Name}"); + } + + break; + default: + Contract.Fail("Unknown property type"); + break; + } + } + + private static class ValidateAssert + { + /// Validate two values are equal. + /// Type of the values to compare. + /// The left value to compare. + /// The right value to compare. + /// Diagnostic message if the comparison fails. + public static void AreEqual(T left, T right, string message) + { + if (!left.Equals(right)) + { + throw new SchemaException(message); + } + } + + /// Validate a predicate is true. + /// The predicate to check. + /// Diagnostic message if the comparison fails. + public static void IsTrue(bool predicate, string message) + { + if (!predicate) + { + throw new SchemaException(message); + } + } + + /// + /// Validate contains only characters valid in a schema + /// identifier. + /// + /// The identifier to check. + /// Diagnostic label describing . + public static void IsValidIdentifier(string identifier, string label) + { + if (string.IsNullOrWhiteSpace(identifier)) + { + throw new SchemaException($"{label} must be a valid identifier: {identifier}"); + } + } + + /// Validate is a valid . + /// The id to check. + /// Diagnostic label describing . + public static void IsValidSchemaId(SchemaId id, string label) + { + if (id == SchemaId.Invalid) + { + throw new SchemaException($"{label} cannot be 0"); + } + } + + /// Validate does not already appear within the given scope. + /// The type of the keys within the scope. + /// The type of the values within the scope. + /// The key to check. + /// The value to add to the scope if there is no duplicate. + /// The set of existing values within the scope. + /// Diagnostic label describing . + /// Diagnostic label describing . + public static void DuplicateCheck(TKey key, TValue value, Dictionary scope, string label, string scopeLabel) + { + if (scope.ContainsKey(key)) + { + throw new SchemaException($"{label} must be unique within a {scopeLabel}: {key}"); + } + + scope.Add(key, value); + } + + /// Validate does appear within the given scope. + /// The type of the keys within the scope. + /// The type of the values within the scope. + /// The key to check. + /// The set of existing values within the scope. + /// Diagnostic label describing . + /// Diagnostic label describing . + public static TValue Exists(TKey key, Dictionary scope, string label, string scopeLabel) + { + if (!scope.TryGetValue(key, out TValue value)) + { + throw new SchemaException($"{label} must exist within a {scopeLabel}: {key}"); + } + + return value; + } + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/ScopePropertyType.cs b/dotnet/src/HybridRow/Schemas/ScopePropertyType.cs new file mode 100644 index 0000000..b41b730 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/ScopePropertyType.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + public abstract class ScopePropertyType : PropertyType + { + /// True if the property's child elements cannot be mutated in place. + /// Immutable properties can still be replaced in their entirety. + [JsonProperty(PropertyName = "immutable")] + [JsonConverter(typeof(StrictBooleanConverter))] + public bool Immutable { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/SetPropertyType.cs b/dotnet/src/HybridRow/Schemas/SetPropertyType.cs new file mode 100644 index 0000000..3532056 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SetPropertyType.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// Set properties represent an unbounded set of zero or more unique items. + /// + /// Sets may be typed or untyped. Within typed sets, all items MUST be the same type. The + /// type of items is specified via . Typed sets may be stored more efficiently + /// than untyped sets. When is unspecified, the set is untyped and its items + /// may be heterogeneous. + /// + /// Each item within a set must be unique. Uniqueness is defined by the HybridRow encoded sequence + /// of bytes for the item. + /// + public class SetPropertyType : ScopePropertyType + { + /// (Optional) type of the elements of the set, if a typed set, otherwise null. + [JsonProperty(PropertyName = "items")] + public PropertyType Items { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/SortDirection.cs b/dotnet/src/HybridRow/Schemas/SortDirection.cs new file mode 100644 index 0000000..1a2e5f7 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/SortDirection.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + /// Describes the sort order direction. + [JsonConverter(typeof(StringEnumConverter), true)] // camelCase:true + public enum SortDirection + { + /// Sorts from the lowest to the highest value. + [EnumMember(Value = "asc")] + Ascending = 0, + + /// Sorts from the highests to the lowest value. + [EnumMember(Value = "desc")] + Descending, + } +} diff --git a/dotnet/src/HybridRow/Schemas/StaticKey.cs b/dotnet/src/HybridRow/Schemas/StaticKey.cs new file mode 100644 index 0000000..07f0013 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/StaticKey.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// + /// Describes a property or set of properties whose values MUST be the same for all rows that share the same partition key. + /// + public class StaticKey + { + /// The logical path of the referenced property. + /// Static path MUST refer to properties defined within the same . + [JsonProperty(PropertyName = "path", Required = Required.Always)] + public string Path { get; set; } + } +} diff --git a/dotnet/src/HybridRow/Schemas/StorageKind.cs b/dotnet/src/HybridRow/Schemas/StorageKind.cs new file mode 100644 index 0000000..90bd201 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/StorageKind.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + /// Describes the storage placement for primitive properties. + [JsonConverter(typeof(StringEnumConverter), true)] // camelCase:true + public enum StorageKind + { + /// The property defines a sparse column. + /// + /// Columns marked consume no space in the row when not present. When + /// present they appear in an unordered linked list at the end of the row. Access time for + /// columns is proportional to the number of columns in the + /// row. + /// + Sparse = 0, + + /// The property is a fixed-length, space-reserved column. + /// + /// The column will consume 1 null-bit, and its byte-width regardless of whether the value is + /// present in the row. + /// + Fixed, + + /// The property is a variable-length column. + /// + /// The column will consume 1 null-bit regardless of whether the value is present. When the value is + /// present it will also consume a variable number of bytes to encode the length preceding the actual + /// value. + /// + /// When a long value is marked then a null-bit is reserved and + /// the value is optionally encoded as if small enough to fit, otherwise the + /// null-bit is set and the value is encoded as . + /// + /// + Variable, + } +} diff --git a/dotnet/src/HybridRow/Schemas/StrictBooleanConverter.cs b/dotnet/src/HybridRow/Schemas/StrictBooleanConverter.cs new file mode 100644 index 0000000..ebdfb46 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/StrictBooleanConverter.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System; + using System.Diagnostics.CodeAnalysis; + using Newtonsoft.Json; + + public class StrictBooleanConverter : JsonConverter + { + public override bool CanWrite => false; + + public override bool CanConvert(Type objectType) + { + return objectType == typeof(bool); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + switch (reader.TokenType) + { + case JsonToken.Boolean: + return serializer.Deserialize(reader, objectType); + default: + throw new JsonSerializationException($"Token \"{reader.Value}\" of type {reader.TokenType} was not a JSON bool"); + } + } + + [ExcludeFromCodeCoverage] + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/StrictIntegerConverter.cs b/dotnet/src/HybridRow/Schemas/StrictIntegerConverter.cs new file mode 100644 index 0000000..4766ab0 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/StrictIntegerConverter.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.Numerics; + using Newtonsoft.Json; + + public class StrictIntegerConverter : JsonConverter + { + public override bool CanWrite => false; + + public override bool CanConvert(Type objectType) + { + return StrictIntegerConverter.IsIntegerType(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + switch (reader.TokenType) + { + case JsonToken.Integer: + return serializer.Deserialize(reader, objectType); + default: + throw new JsonSerializationException($"Token \"{reader.Value}\" of type {reader.TokenType} was not a JSON integer"); + } + } + + [ExcludeFromCodeCoverage] + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + } + + private static bool IsIntegerType(Type type) + { + if (type == typeof(long) || + type == typeof(ulong) || + type == typeof(int) || + type == typeof(uint) || + type == typeof(short) || + type == typeof(ushort) || + type == typeof(byte) || + type == typeof(sbyte) || + type == typeof(BigInteger)) + { + return true; + } + + return false; + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/TaggedPropertyType.cs b/dotnet/src/HybridRow/Schemas/TaggedPropertyType.cs new file mode 100644 index 0000000..4194c00 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/TaggedPropertyType.cs @@ -0,0 +1,44 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Collections.Generic; + using Newtonsoft.Json; + + /// Tagged properties pair one or more typed values with an API-specific uint8 type code. + /// + /// The uint8 type code is implicitly in position 0 within the resulting tagged and should not + /// be specified in . + /// + public class TaggedPropertyType : ScopePropertyType + { + internal const int MinTaggedArguments = 1; + internal const int MaxTaggedArguments = 2; + + /// Types of the elements of the tagged in element order. + private List items; + + /// Initializes a new instance of the class. + public TaggedPropertyType() + { + this.items = new List(); + } + + /// Types of the elements of the tagged in element order. + [JsonProperty(PropertyName = "items")] + public List Items + { + get + { + return this.items; + } + + set + { + this.items = value ?? new List(); + } + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/TuplePropertyType.cs b/dotnet/src/HybridRow/Schemas/TuplePropertyType.cs new file mode 100644 index 0000000..3abf163 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/TuplePropertyType.cs @@ -0,0 +1,37 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Collections.Generic; + using Newtonsoft.Json; + + /// Tuple properties represent a typed, finite, ordered set of two or more items. + public class TuplePropertyType : ScopePropertyType + { + /// Types of the elements of the tuple in element order. + private List items; + + /// Initializes a new instance of the class. + public TuplePropertyType() + { + this.items = new List(); + } + + /// Types of the elements of the tuple in element order. + [JsonProperty(PropertyName = "items")] + public List Items + { + get + { + return this.items; + } + + set + { + this.items = value ?? new List(); + } + } + } +} diff --git a/dotnet/src/HybridRow/Schemas/TypeKind.cs b/dotnet/src/HybridRow/Schemas/TypeKind.cs new file mode 100644 index 0000000..24fca39 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/TypeKind.cs @@ -0,0 +1,133 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + /// Describes the logical type of a property. + [JsonConverter(typeof(StringEnumConverter), true)] // camelCase:true + public enum TypeKind + { + /// Reserved. + Invalid = 0, + + /// The literal null. + /// + /// When used as a fixed column, only a presence bit is allocated. When used as a sparse + /// column, a sparse value with 0-length payload is written. + /// + Null, + + /// + /// A boolean property. Boolean properties are allocated a single bit when schematized within + /// a row. + /// + [EnumMember(Value = "bool")] + Boolean, + + /// 8-bit signed integer. + Int8, + + /// + /// 16-bit signed integer. + /// + Int16, + + /// 32-bit signed integer. + Int32, + + /// 64-bit signed integer. + Int64, + + /// 8-bit unsigned integer. + [EnumMember(Value = "uint8")] + UInt8, + + /// 16-bit unsigned integer. + [EnumMember(Value = "uint16")] + UInt16, + + /// 32-bit unsigned integer. + [EnumMember(Value = "uint32")] + UInt32, + + /// 64-bit unsigned integer. + [EnumMember(Value = "uint64")] + UInt64, + + /// Variable length encoded signed integer. + [EnumMember(Value = "varint")] + VarInt, + + /// Variable length encoded unsigned integer. + [EnumMember(Value = "varuint")] + VarUInt, + + /// 32-bit IEEE 754 floating point value. + Float32, + + /// 64-bit IEEE 754 floating point value. + Float64, + + /// 128-bit IEEE 754-2008 floating point value. + Float128, + + /// 128-bit floating point value. See + Decimal, + + /// + /// 64-bit date/time value in 100ns increments from C# epoch. See + /// + [EnumMember(Value = "datetime")] + DateTime, + + /// + /// 64-bit date/time value in milliseconds increments from Unix epoch. See + /// + [EnumMember(Value = "unixdatetime")] + UnixDateTime, + + /// 128-bit globally unique identifier (in little-endian byte order). + Guid, + + /// 12-byte MongoDB Object Identifier (in little-endian byte order). + [EnumMember(Value = "mongodbobjectid")] + MongoDbObjectId, + + /// Zero to MAX_ROW_SIZE bytes encoded as UTF-8 code points. + Utf8, + + /// Zero to MAX_ROW_SIZE untyped bytes. + Binary, + + /// An object property. + Object, + + /// An array property, either typed or untyped. + Array, + + /// A set property, either typed or untyped. + Set, + + /// A map property, either typed or untyped. + Map, + + /// A tuple property. Tuples are typed, finite, ordered, sets. + Tuple, + + /// A tagged property. Tagged properties pair one or more typed values with an API-specific uint8 type code. + Tagged, + + /// A row with schema. + /// May define either a top-level table schema or a UDT (nested row). + Schema, + + /// An untyped sparse field. + /// May only be used to define the type within a nested scope (e.g. or . + Any, + } +} diff --git a/dotnet/src/HybridRow/Schemas/UdtPropertyType.cs b/dotnet/src/HybridRow/Schemas/UdtPropertyType.cs new file mode 100644 index 0000000..d1ba9d9 --- /dev/null +++ b/dotnet/src/HybridRow/Schemas/UdtPropertyType.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas +{ + using Newtonsoft.Json; + + /// UDT properties represent nested structures with an independent schema. + /// + /// UDT properties include a nested row within an existing row as a column. The schema of the + /// nested row may be evolved independently of the outer row. Changes to the independent schema affect + /// all outer schemas where the UDT is used. + /// + public class UdtPropertyType : ScopePropertyType + { + public UdtPropertyType() + { + this.SchemaId = SchemaId.Invalid; + } + + /// The identifier of the UDT schema defining the structure for the nested row. + /// + /// The UDT schema MUST be defined within the same as the schema that + /// references it. + /// + [JsonProperty(PropertyName = "name", Required = Required.Always)] + public string Name { get; set; } + + /// The unique identifier for a schema. + /// + /// Optional uniquifier if multiple versions of appears within the Namespace. + ///

+ /// If multiple versions of a UDT are defined within the then the globally + /// unique identifier of the specific version referenced MUST be provided. + ///

+ /// + [JsonProperty(PropertyName = "id", Required = Required.DisallowNull, DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public SchemaId SchemaId { get; set; } + } +} diff --git a/dotnet/src/HybridRow/SystemSchemas/SystemSchema.json b/dotnet/src/HybridRow/SystemSchemas/SystemSchema.json new file mode 100644 index 0000000..bae75f3 --- /dev/null +++ b/dotnet/src/HybridRow/SystemSchemas/SystemSchema.json @@ -0,0 +1,54 @@ +// HybridRow RecordIO Schema +{ + "name": "Microsoft.Azure.Cosmos.HybridRow.RecordIO", + "version": "v1", + "schemas": [ + { + "name": "EmptySchema", + "id": 2147473650, + "type": "schema", + "properties": [] + }, + { + "name": "Segment", + "id": 2147473648, + "type": "schema", + "properties": [ + { + "path": "length", + "type": { "type": "int32", "storage": "fixed" }, + "comment": + "(Required) length (in bytes) of this RecordIO segment header itself. Does NOT include the length of the records that follow." + }, + { + "path": "comment", + "type": { "type": "utf8", "storage": "sparse" }, + "comment": "A comment describing the data in this RecordIO segment." + }, + { + // TODO: this should be converted to a HybridRow UDT instead. + "path": "sdl", + "type": { "type": "utf8", "storage": "sparse" }, + "comment": "A HybridRow Schema in SDL (json-format)." + } + ] + }, + { + "name": "Record", + "id": 2147473649, + "type": "schema", + "properties": [ + { + "path": "length", + "type": { "type": "int32", "storage": "fixed", "nullable": false }, + "comment": "(Required) length (in bytes) of the HybridRow value that follows this record header." + }, + { + "path": "crc32", + "type": { "type": "uint32", "storage": "fixed", "nullable": false }, + "comment": "(Optional) CRC-32 as described in ISO 3309." + } + ] + } + ] +} diff --git a/dotnet/src/HybridRow/UnixDateTime.cs b/dotnet/src/HybridRow/UnixDateTime.cs new file mode 100644 index 0000000..d1aa49e --- /dev/null +++ b/dotnet/src/HybridRow/UnixDateTime.cs @@ -0,0 +1,77 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRow +{ + using System; + using System.Diagnostics; + using System.Runtime.InteropServices; + + /// A wall clock time expressed in milliseconds since the Unix Epoch. + /// + /// A is a fixed length value-type providing millisecond + /// granularity as a signed offset from the Unix Epoch (midnight, January 1, 1970 UTC). + /// + [DebuggerDisplay("{" + nameof(UnixDateTime.Milliseconds) + "}")] + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public readonly struct UnixDateTime : IEquatable + { + /// The size (in bytes) of a UnixDateTime. + public const int Size = sizeof(long); + + /// The unix epoch. + /// values are signed values centered on . + /// + /// This is the same value as default(). + public static readonly UnixDateTime Epoch = default(UnixDateTime); + + /// Initializes a new instance of the struct. + /// The number of milliseconds since . + public UnixDateTime(long milliseconds) + { + this.Milliseconds = milliseconds; + } + + /// The number of milliseconds since . + /// This value may be negative. + public long Milliseconds { get; } + + /// Operator == overload. + public static bool operator ==(UnixDateTime left, UnixDateTime right) + { + return left.Equals(right); + } + + /// Operator != overload. + public static bool operator !=(UnixDateTime left, UnixDateTime right) + { + return !left.Equals(right); + } + + /// Returns true if this is the same value as . + /// The value to compare against. + /// True if the two values are the same. + public bool Equals(UnixDateTime other) + { + return this.Milliseconds == other.Milliseconds; + } + + /// overload. + public override bool Equals(object obj) + { + if (object.ReferenceEquals(null, obj)) + { + return false; + } + + return obj is UnixDateTime && this.Equals((UnixDateTime)obj); + } + + /// overload. + public override int GetHashCode() + { + return this.Milliseconds.GetHashCode(); + } + } +} diff --git a/dotnet/src/HybridRowCLI/App.config b/dotnet/src/HybridRowCLI/App.config new file mode 100644 index 0000000..f6888fa --- /dev/null +++ b/dotnet/src/HybridRowCLI/App.config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/dotnet/src/HybridRowCLI/CompileCommand.cs b/dotnet/src/HybridRowCLI/CompileCommand.cs new file mode 100644 index 0000000..e0b9976 --- /dev/null +++ b/dotnet/src/HybridRowCLI/CompileCommand.cs @@ -0,0 +1,82 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Extensions.CommandLineUtils; + + public class CompileCommand + { + private CompileCommand() + { + } + + public bool Verbose { get; set; } = false; + + public List Schemas { get; set; } = null; + + public static void AddCommand(CommandLineApplication app) + { + app.Command( + "compile", + command => + { + command.Description = "Compile and validate a hybrid row schema."; + command.ExtendedHelpText = "Compile a hybrid row schema using the object model compiler and run schema consistency validator."; + command.HelpOption("-? | -h | --help"); + + CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output. Default: false.", CommandOptionType.NoValue); + CommandArgument schemasOpt = command.Argument( + "schema", + "File(s) containing the schema namespace to compile.", + arg => { arg.MultipleValues = true; }); + + command.OnExecute( + () => + { + CompileCommand config = new CompileCommand(); + config.Verbose = verboseOpt.HasValue(); + config.Schemas = schemasOpt.Values; + + return config.OnExecute(); + }); + }); + } + + public int OnExecute() + { + foreach (string schemaFile in this.Schemas) + { + if (this.Verbose) + { + Console.WriteLine($"Compiling {schemaFile}..."); + Console.WriteLine(); + } + + string json = File.ReadAllText(schemaFile); + Namespace n = Namespace.Parse(json); + if (this.Verbose) + { + Console.WriteLine($"Namespace: {n.Name}"); + foreach (Schema s in n.Schemas) + { + Console.WriteLine($" {s.SchemaId} Schema: {s.Name}"); + } + } + + if (this.Verbose) + { + Console.WriteLine(); + Console.WriteLine($"Compiling {schemaFile} complete.\n"); + } + } + + return 0; + } + } +} diff --git a/dotnet/src/HybridRowCLI/ConsoleNative.cs b/dotnet/src/HybridRowCLI/ConsoleNative.cs new file mode 100644 index 0000000..65ba202 --- /dev/null +++ b/dotnet/src/HybridRowCLI/ConsoleNative.cs @@ -0,0 +1,78 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable CA1028 // Enum Storage should be Int32 +#pragma warning disable CA1707 // Identifiers should not contain underscores + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using System.Runtime.InteropServices; + + public static class ConsoleNative + { + [Flags] + public enum ConsoleModes : uint + { + ENABLE_PROCESSED_INPUT = 0x0001, + ENABLE_LINE_INPUT = 0x0002, + ENABLE_ECHO_INPUT = 0x0004, + ENABLE_WINDOW_INPUT = 0x0008, + ENABLE_MOUSE_INPUT = 0x0010, + ENABLE_INSERT_MODE = 0x0020, + ENABLE_QUICK_EDIT_MODE = 0x0040, + ENABLE_EXTENDED_FLAGS = 0x0080, + ENABLE_AUTO_POSITION = 0x0100, + + ENABLE_PROCESSED_OUTPUT = 0x0001, + ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002, + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004, + DISABLE_NEWLINE_AUTO_RETURN = 0x0008, + ENABLE_LVB_GRID_WORLDWIDE = 0x0010, + } + + private const int StdOutputHandle = -11; + + public static ConsoleModes Mode + { + get + { + ConsoleNative.GetConsoleMode(ConsoleNative.GetStdHandle(ConsoleNative.StdOutputHandle), out ConsoleModes mode); + return mode; + } + + set => ConsoleNative.SetConsoleMode(ConsoleNative.GetStdHandle(ConsoleNative.StdOutputHandle), value); + } + + public static void SetBufferSize(int width, int height) + { + Coord size = new Coord + { + X = (short)width, + Y = (short)height, + }; + + ConsoleNative.SetConsoleScreenBufferSize(ConsoleNative.GetStdHandle(ConsoleNative.StdOutputHandle), size); + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr GetStdHandle(int nStdHandle); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool SetConsoleScreenBufferSize(IntPtr hConsoleHandle, Coord dwSize); + + [StructLayout(LayoutKind.Sequential)] + private struct Coord + { + public short X; + public short Y; + } + } +} diff --git a/dotnet/src/HybridRowCLI/Csv2HybridRowCommand.cs b/dotnet/src/HybridRowCLI/Csv2HybridRowCommand.cs new file mode 100644 index 0000000..79306be --- /dev/null +++ b/dotnet/src/HybridRowCLI/Csv2HybridRowCommand.cs @@ -0,0 +1,474 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.Extensions.CommandLineUtils; + + public class Csv2HybridRowCommand + { + private const int InitialCapacity = 2 * 1024 * 1024; + + private bool verbose; + private long limit; + private string csvFile; + private string outputFile; + private string namespaceFile; + private string tableName; + + private Csv2HybridRowCommand() + { + } + + public static void AddCommand(CommandLineApplication app) + { + app.Command( + "csv2hr", + command => + { + command.Description = "Convert a CSV data set to hybrid row."; + command.ExtendedHelpText = + "Convert textual Comma-Separated-Values data set into a HybridRow.\n\n" + + "The csv2hr command accepts CSV files in the following format:\n" + + "\t* First line contains column names, followed by 0 or more blank rows.\n" + + "\t* All subsequent lines contain rows. A row with too few values treats \n" + + "\t all missing values as the empty string. Lines consistent entirely of \n" + + "\t whitespace are omitted. Columns without a name are discarded."; + + command.HelpOption("-? | -h | --help"); + + CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue); + + CommandOption limitOpt = command.Option("--limit", "Limit the number of input rows processed.", CommandOptionType.SingleValue); + + CommandOption namespaceOpt = command.Option( + "-n|--namespace", + "File containing the schema namespace.", + CommandOptionType.SingleValue); + + CommandOption tableNameOpt = command.Option( + "-tn|--tablename", + "The table schema (when using -namespace). Default: null.", + CommandOptionType.SingleValue); + + CommandArgument csvOpt = command.Argument("csv", "File containing csv to convert."); + CommandArgument outputOpt = command.Argument("output", "Output file to contain the HybridRow conversion."); + + command.OnExecute( + () => + { + Csv2HybridRowCommand config = new Csv2HybridRowCommand + { + verbose = verboseOpt.HasValue(), + limit = !limitOpt.HasValue() ? long.MaxValue : long.Parse(limitOpt.Value()), + namespaceFile = namespaceOpt.Value().Trim(), + tableName = tableNameOpt.Value().Trim(), + csvFile = csvOpt.Value.Trim(), + outputFile = outputOpt.Value.Trim() + }; + + if (string.IsNullOrWhiteSpace(config.csvFile)) + { + throw new CommandParsingException(command, "Error: Input file MUST be provided."); + } + + if (string.IsNullOrWhiteSpace(config.outputFile)) + { + throw new CommandParsingException(command, "Error: Output file MUST be provided."); + } + + if (config.csvFile == config.outputFile) + { + throw new CommandParsingException(command, "Error: Input and Output files MUST be different."); + } + + return config.OnExecuteAsync().Result; + }); + }); + } + + private static Result ProcessLine( + Utf8String[] paths, + string line, + LayoutResolver resolver, + Layout layout, + MemorySpanResizer resizer, + out ReadOnlyMemory record) + { + Contract.Requires(paths != null); + Contract.Requires(line != null); + + RowBuffer row = new RowBuffer(resizer.Memory.Length, resizer); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + WriterContext ctx = new WriterContext { Paths = paths, Line = line }; + Result r = RowWriter.WriteBuffer(ref row, ctx, Csv2HybridRowCommand.WriteLine); + if (r != Result.Success) + { + record = default; + return r; + } + + record = resizer.Memory.Slice(0, row.Length); + return Result.Success; + } + + private static Result WriteLine(ref RowWriter writer, TypeArgument typeArg, WriterContext ctx) + { + ReadOnlySpan remaining = ctx.Line.AsSpan(); + foreach (Utf8String path in ctx.Paths) + { + int comma = remaining.IndexOf(','); + ReadOnlySpan fieldValue = comma == -1 ? remaining : remaining.Slice(0, comma); + remaining = comma == -1 ? ReadOnlySpan.Empty : remaining.Slice(comma + 1); + Result r = Csv2HybridRowCommand.WriteField(ref writer, path, fieldValue); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + private static Result WriteField(ref RowWriter writer, Utf8String path, ReadOnlySpan fieldValue) + { + writer.Layout.TryFind(path, out LayoutColumn col); + switch (col?.Type?.LayoutCode ?? LayoutCode.Invalid) + { + case LayoutCode.Boolean: + { + if (!bool.TryParse(fieldValue.AsString(), out bool value)) + { + goto default; + } + + return writer.WriteBool(path, value); + } + + case LayoutCode.Int8: + { + if (!sbyte.TryParse(fieldValue.AsString(), out sbyte value)) + { + goto default; + } + + return writer.WriteInt8(path, value); + } + + case LayoutCode.Int16: + { + if (!short.TryParse(fieldValue.AsString(), out short value)) + { + goto default; + } + + return writer.WriteInt16(path, value); + } + + case LayoutCode.Int32: + { + if (!int.TryParse(fieldValue.AsString(), out int value)) + { + goto default; + } + + return writer.WriteInt32(path, value); + } + + case LayoutCode.Int64: + { + if (!long.TryParse(fieldValue.AsString(), out long value)) + { + goto default; + } + + return writer.WriteInt64(path, value); + } + + case LayoutCode.UInt8: + { + if (!byte.TryParse(fieldValue.AsString(), out byte value)) + { + goto default; + } + + return writer.WriteUInt8(path, value); + } + + case LayoutCode.UInt16: + { + if (!ushort.TryParse(fieldValue.AsString(), out ushort value)) + { + goto default; + } + + return writer.WriteUInt16(path, value); + } + + case LayoutCode.UInt32: + { + if (!uint.TryParse(fieldValue.AsString(), out uint value)) + { + goto default; + } + + return writer.WriteUInt32(path, value); + } + + case LayoutCode.UInt64: + { + if (!ulong.TryParse(fieldValue.AsString(), out ulong value)) + { + goto default; + } + + return writer.WriteUInt64(path, value); + } + + case LayoutCode.VarInt: + { + if (!long.TryParse(fieldValue.AsString(), out long value)) + { + goto default; + } + + return writer.WriteVarInt(path, value); + } + + case LayoutCode.VarUInt: + { + if (!ulong.TryParse(fieldValue.AsString(), out ulong value)) + { + goto default; + } + + return writer.WriteVarUInt(path, value); + } + + case LayoutCode.Float32: + { + if (!float.TryParse(fieldValue.AsString(), out float value)) + { + goto default; + } + + return writer.WriteFloat32(path, value); + } + + case LayoutCode.Float64: + { + if (!double.TryParse(fieldValue.AsString(), out double value)) + { + goto default; + } + + return writer.WriteFloat64(path, value); + } + + case LayoutCode.Decimal: + { + if (!decimal.TryParse(fieldValue.AsString(), out decimal value)) + { + goto default; + } + + return writer.WriteDecimal(path, value); + } + + case LayoutCode.DateTime: + { + IFormatProvider provider = CultureInfo.CurrentCulture; + DateTimeStyles style = DateTimeStyles.AdjustToUniversal | + DateTimeStyles.AllowWhiteSpaces | + DateTimeStyles.AssumeUniversal; + + if (!DateTime.TryParse(fieldValue.AsString(), provider, style, out DateTime value)) + { + goto default; + } + + return writer.WriteDateTime(path, value); + } + + case LayoutCode.Guid: + { + string s = fieldValue.AsString(); + + // If the guid is quoted then remove the quotes. + if (s.Length > 2 && s[0] == '"' && s[s.Length - 1] == '"') + { + s = s.Substring(1, s.Length - 2); + } + + if (!Guid.TryParse(s, out Guid value)) + { + goto default; + } + + return writer.WriteGuid(path, value); + } + + case LayoutCode.Binary: + { + try + { + byte[] newBytes = Convert.FromBase64String(fieldValue.AsString()); + return writer.WriteBinary(path, newBytes.AsSpan()); + } + catch (Exception) + { + // fall through and try hex. + } + + if (fieldValue.TryParseHexString(out byte[] fromHexBytes)) + { + return writer.WriteBinary(path, fromHexBytes); + } + + goto default; + } + + case LayoutCode.UnixDateTime: + { + if (!long.TryParse(fieldValue.AsString(), out long value)) + { + goto default; + } + + return writer.WriteUnixDateTime(path, new UnixDateTime(value)); + } + + case LayoutCode.MongoDbObjectId: + { + string s = fieldValue.AsString(); + try + { + byte[] newBytes = Convert.FromBase64String(s); + return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(newBytes.AsSpan())); + } + catch (Exception) + { + // fall through and try hex. + } + + if (fieldValue.TryParseHexString(out byte[] fromHexBytes)) + { + return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(fromHexBytes)); + } + + goto default; + } + + default: + return writer.WriteString(path, fieldValue.AsString()); + } + } + + private async Task OnExecuteAsync() + { + LayoutResolver resolver = await SchemaUtil.CreateResolverAsync(this.namespaceFile, this.verbose); + string sdl = string.IsNullOrWhiteSpace(this.namespaceFile) ? null : await File.ReadAllTextAsync(this.namespaceFile); + MemorySpanResizer resizer = new MemorySpanResizer(Csv2HybridRowCommand.InitialCapacity); + using (Stream stm = new FileStream(this.csvFile, FileMode.Open)) + using (TextReader txt = new StreamReader(stm)) + using (Stream outputStm = new FileStream(this.outputFile, FileMode.Create)) + { + // Read the CSV "schema". + string fieldNamesLine = await txt.ReadLineAsync(); + while (fieldNamesLine != null && string.IsNullOrWhiteSpace(fieldNamesLine)) + { + fieldNamesLine = await txt.ReadLineAsync(); + } + + if (fieldNamesLine == null) + { + await Console.Error.WriteLineAsync($"Input file contains no schema: {this.csvFile}"); + return -1; + } + + string[] fieldNames = fieldNamesLine.Split(','); + if (!(from c in fieldNames where !string.IsNullOrWhiteSpace(c) select c).Any()) + { + await Console.Error.WriteLineAsync($"All columns are ignored. Does this file have field headers?: {this.csvFile}"); + return -1; + } + + Utf8String[] paths = (from c in fieldNames select Utf8String.TranscodeUtf16(c)).ToArray(); + + SchemaId tableId = SystemSchema.EmptySchemaId; + if (!string.IsNullOrWhiteSpace(this.tableName)) + { + tableId = (resolver as LayoutResolverNamespace)?.Namespace?.Schemas?.Find(s => s.Name == this.tableName)?.SchemaId ?? + SchemaId.Invalid; + + if (tableId == SchemaId.Invalid) + { + await Console.Error.WriteLineAsync($"Error: schema {this.tableName} could not be found in {this.namespaceFile}."); + return -1; + } + } + + Layout layout = resolver.Resolve(tableId); + + long totalWritten = 0; + Segment segment = new Segment("CSV conversion from HybridRowCLI csv2hr", sdl); + Result r = await outputStm.WriteRecordIOAsync( + segment, + async index => + { + if (totalWritten >= this.limit) + { + return (Result.Success, default); + } + + string line = await txt.ReadLineAsync(); + while (line != null && string.IsNullOrWhiteSpace(line)) + { + line = await txt.ReadLineAsync(); + } + + if (line == null) + { + return (Result.Success, default); + } + + Result r2 = Csv2HybridRowCommand.ProcessLine(paths, line, resolver, layout, resizer, out ReadOnlyMemory record); + if (r2 != Result.Success) + { + return (r2, default); + } + + totalWritten++; + return (r2, record); + }); + + if (r != Result.Success) + { + Console.WriteLine($"Error while writing record: {totalWritten} to HybridRow(s): {this.outputFile}"); + return -1; + } + + Console.WriteLine($"Wrote ({totalWritten}) HybridRow(s): {this.outputFile}"); + return 0; + } + } + + private struct WriterContext + { + public Utf8String[] Paths; + public string Line; + } + } +} diff --git a/dotnet/src/HybridRowCLI/HybridRowCLIProgram.cs b/dotnet/src/HybridRowCLI/HybridRowCLIProgram.cs new file mode 100644 index 0000000..d095462 --- /dev/null +++ b/dotnet/src/HybridRowCLI/HybridRowCLIProgram.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using Microsoft.Extensions.CommandLineUtils; + + internal class HybridRowCLIProgram + { + private const string Version = "1.0.0"; + private const string LongVersion = nameof(HybridRowCLI) + " " + HybridRowCLIProgram.Version; + + public static int Main(string[] args) + { + try + { + CommandLineApplication command = new CommandLineApplication(throwOnUnexpectedArg: true) + { + Name = nameof(HybridRowCLI), + Description = "HybridRow Command Line Interface.", + }; + + command.HelpOption("-? | -h | --help"); + command.VersionOption("-ver | --version", HybridRowCLIProgram.Version, HybridRowCLIProgram.LongVersion); + CompileCommand.AddCommand(command); + PrintCommand.AddCommand(command); + Json2HybridRowCommand.AddCommand(command); + Csv2HybridRowCommand.AddCommand(command); + + return command.Execute(args); + } + catch (CommandParsingException ex) + { + Console.Error.WriteLine(ex.Message); + return -1; + } + } + } +} diff --git a/dotnet/src/HybridRowCLI/Json2HybridRowCommand.cs b/dotnet/src/HybridRowCLI/Json2HybridRowCommand.cs new file mode 100644 index 0000000..9086c37 --- /dev/null +++ b/dotnet/src/HybridRowCLI/Json2HybridRowCommand.cs @@ -0,0 +1,710 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using System.Globalization; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.Extensions.CommandLineUtils; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + public class Json2HybridRowCommand + { + private const int InitialCapacity = 2 * 1024 * 1024; + + private Json2HybridRowCommand() + { + } + + public bool Verbose { get; set; } + + public string JsonFile { get; set; } + + public string OutputFile { get; set; } + + public string NamespaceFile { get; set; } + + public string TableName { get; set; } + + public static void AddCommand(CommandLineApplication app) + { + app.Command( + "json2hr", + command => + { + command.Description = "Convert a JSON document to hybrid row."; + command.ExtendedHelpText = + "Convert textual JSON document into a HybridRow.\n\n" + + "The json2hr command accepts files in two formats:\n" + + "\t* A JSON file whose top-level element is a JSON Object. This file is converted to a HybridRow binary " + + "\t file containing exactly 1 record.\n" + + "\t* A JSON file whose top-level element is a JSON Array. This file is convert to a A HybridRow RecordIO " + + "\t file containing 0 or more records, one for each element of the JSON Array."; + + command.HelpOption("-? | -h | --help"); + + CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue); + + CommandOption namespaceOpt = command.Option( + "-n|--namespace", + "File containing the schema namespace.", + CommandOptionType.SingleValue); + + CommandOption tableNameOpt = command.Option( + "-tn|--tablename", + "The table schema (when using -namespace). Default: null.", + CommandOptionType.SingleValue); + + CommandArgument jsonOpt = command.Argument("json", "File containing json to convert."); + CommandArgument outputOpt = command.Argument("output", "Output file to contain the HybridRow conversion."); + + command.OnExecute( + () => + { + Json2HybridRowCommand config = new Json2HybridRowCommand + { + Verbose = verboseOpt.HasValue(), + NamespaceFile = namespaceOpt.Value(), + TableName = tableNameOpt.Value(), + JsonFile = jsonOpt.Value, + OutputFile = outputOpt.Value + }; + + return config.OnExecuteAsync().Result; + }); + }); + } + + public async Task OnExecuteAsync() + { + LayoutResolver globalResolver = await SchemaUtil.CreateResolverAsync(this.NamespaceFile, this.Verbose); + string sdl = string.IsNullOrWhiteSpace(this.NamespaceFile) ? null : await File.ReadAllTextAsync(this.NamespaceFile); + MemorySpanResizer resizer = new MemorySpanResizer(Json2HybridRowCommand.InitialCapacity); + using (Stream stm = new FileStream(this.JsonFile, FileMode.Open)) + using (TextReader txt = new StreamReader(stm)) + using (JsonReader reader = new JsonTextReader(txt)) + { + // Turn off any special parsing conversions. Just raw JSON tokens. + reader.DateParseHandling = DateParseHandling.None; + reader.FloatParseHandling = FloatParseHandling.Double; + + if (!await reader.ReadAsync()) + { + await Console.Error.WriteLineAsync("Error: file is empty."); + return -1; + } + + switch (reader.TokenType) + { + case JsonToken.StartObject: + { + JObject obj = await JObject.LoadAsync(reader); + Result r = this.ProcessObject(obj, globalResolver, resizer, out ReadOnlyMemory record); + if (r != Result.Success) + { + Console.WriteLine($"Error while writing record: 0 to HybridRow(s): {this.OutputFile}"); + return -1; + } + + // Write the output. + using (Stream outputStm = new FileStream(this.OutputFile, FileMode.Create)) + { + await outputStm.WriteAsync(record); + Console.WriteLine($"Wrote (1) HybridRow: {this.OutputFile}"); + } + + return 0; + } + + case JsonToken.StartArray: + { + using (Stream outputStm = new FileStream(this.OutputFile, FileMode.Create)) + { + long totalWritten = 0; + Segment segment = new Segment("JSON conversion from HybridRowCLI json2hr", sdl); + Result r = await outputStm.WriteRecordIOAsync( + segment, + async index => + { + if (!await reader.ReadAsync() || reader.TokenType == JsonToken.EndArray) + { + return (Result.Success, default); + } + + JObject obj = await JObject.LoadAsync(reader); + Result r2 = this.ProcessObject(obj, globalResolver, resizer, out ReadOnlyMemory record); + if (r2 != Result.Success) + { + return (r2, default); + } + + totalWritten++; + return (r2, record); + }); + + if (r != Result.Success) + { + Console.WriteLine($"Error while writing record: {totalWritten} to HybridRow(s): {this.OutputFile}"); + return -1; + } + + Console.WriteLine($"Wrote ({totalWritten}) HybridRow(s): {this.OutputFile}"); + } + + return 0; + } + + default: + await Console.Error.WriteLineAsync("Error: Only JSON documents with top-level Object/Array supported."); + return -1; + } + } + } + + private static string TrimPath(JContainer parent, string path) + { + if (string.IsNullOrEmpty(parent.Path)) + { + return path; + } + + return path.Substring(parent.Path.Length + 1); + } + + /// Returns true if the type code indicates a linear scope (i.e. array like). + /// The scope type code. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsLinearScope(LayoutCode code) + { + if (code < LayoutCode.ObjectScope || code >= LayoutCode.EndScope) + { + return false; + } + + const ulong bitmask = + (0x1UL << (int)(LayoutCode.ArrayScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableArrayScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.TypedArrayScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableTypedArrayScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.TypedSetScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableTypedSetScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.TypedMapScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableTypedMapScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.TypedTupleScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableTypedTupleScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.TaggedScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableTaggedScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.Tagged2Scope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableTagged2Scope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.NullableScope - LayoutCode.ObjectScope)) | + (0x1UL << (int)(LayoutCode.ImmutableNullableScope - LayoutCode.ObjectScope)) + ; + + return ((0x1UL << (code - LayoutCode.ObjectScope)) & bitmask) != 0; + } + + private Result ProcessObject(JObject obj, LayoutResolver resolver, MemorySpanResizer resizer, out ReadOnlyMemory record) + { + Contract.Requires(obj.Type == JTokenType.Object); + SchemaId tableId = SystemSchema.EmptySchemaId; + if (!string.IsNullOrWhiteSpace(this.TableName)) + { + tableId = (resolver as LayoutResolverNamespace)?.Namespace?.Schemas?.Find(s => s.Name == this.TableName)?.SchemaId ?? + SchemaId.Invalid; + + if (tableId == SchemaId.Invalid) + { + Console.Error.WriteLine($"Error: schema {this.TableName} could not be found in {this.NamespaceFile}."); + record = default; + return Result.Failure; + } + } + + Layout layout = resolver.Resolve(tableId); + RowBuffer row = new RowBuffer(resizer.Memory.Length, resizer); + row.InitLayout(HybridRowVersion.V1, layout, resolver); + WriterContext ctx = new WriterContext { Token = obj, PathPrefix = "" }; + Result r = RowWriter.WriteBuffer(ref row, ctx, this.WriteObject); + if (r != Result.Success) + { + record = default; + return r; + } + + record = resizer.Memory.Slice(0, row.Length); + return Result.Success; + } + + private Result WriteArray(ref RowWriter writer, TypeArgument typeArg, WriterContext ctx) + { + JArray obj = ctx.Token as JArray; + TypeArgument propTypeArgument = typeArg.TypeArgs.Count > 0 ? typeArg.TypeArgs[0] : default; + foreach (JToken token in obj.Children()) + { + Result r = this.WriteJToken(ref writer, ctx.PathPrefix, null, propTypeArgument, token); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + private Result WriteObject(ref RowWriter writer, TypeArgument typeArg, WriterContext ctx) + { + JObject obj = ctx.Token as JObject; + Layout layout = writer.Layout; + if (typeArg.TypeArgs.SchemaId != SchemaId.Invalid) + { + layout = writer.Resolver.Resolve(typeArg.TypeArgs.SchemaId); + ctx.PathPrefix = ""; + } + + foreach (JProperty p in obj.Properties()) + { + if (!p.HasValues) + { + continue; + } + + string path = Json2HybridRowCommand.TrimPath(obj, p.Path); + TypeArgument propTypeArg = default; + string propFullPath = ctx.PathPrefix + path; + if (layout.TryFind(propFullPath, out LayoutColumn col)) + { + propTypeArg = col.TypeArg; + } + + Result r = this.WriteJToken(ref writer, ctx.PathPrefix, path, propTypeArg, p.Value); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + + private Result WriteJToken(ref RowWriter writer, string pathPrefix, string path, TypeArgument typeArg, JToken token) + { + TypeArgument propTypeArg; + LayoutCode code = typeArg.Type?.LayoutCode ?? LayoutCode.Invalid; + switch (token.Type) + { + case JTokenType.Object: + propTypeArg = typeArg.Type is LayoutUDT ? typeArg : new TypeArgument(LayoutType.Object); + WriterContext ctx1 = new WriterContext { Token = token.Value(), PathPrefix = $"{pathPrefix}{path}." }; + return writer.WriteScope(path, propTypeArg, ctx1, this.WriteObject); + case JTokenType.Array: + propTypeArg = Json2HybridRowCommand.IsLinearScope(typeArg.Type?.LayoutCode ?? LayoutCode.Invalid) + ? typeArg + : new TypeArgument(LayoutType.Array); + + WriterContext ctx2 = new WriterContext { Token = token.Value(), PathPrefix = $"{pathPrefix}{path}[]." }; + return writer.WriteScope(path, propTypeArg, ctx2, this.WriteArray); + case JTokenType.Integer: + { + long value = token.Value(); + try + { + switch (code) + { + case LayoutCode.Int8: + return writer.WriteInt8(path, (sbyte)value); + case LayoutCode.Int16: + return writer.WriteInt16(path, (short)value); + case LayoutCode.Int32: + return writer.WriteInt32(path, (int)value); + case LayoutCode.Int64: + return writer.WriteInt64(path, value); + case LayoutCode.UInt8: + return writer.WriteUInt8(path, (byte)value); + case LayoutCode.UInt16: + return writer.WriteUInt16(path, (ushort)value); + case LayoutCode.UInt32: + return writer.WriteUInt32(path, (uint)value); + case LayoutCode.UInt64: + return writer.WriteUInt64(path, (ulong)value); + case LayoutCode.VarInt: + return writer.WriteVarInt(path, value); + case LayoutCode.VarUInt: + return writer.WriteVarUInt(path, (ulong)value); + case LayoutCode.Float32: + return writer.WriteFloat32(path, value); + case LayoutCode.Float64: + return writer.WriteFloat64(path, value); + case LayoutCode.Decimal: + return writer.WriteDecimal(path, value); + case LayoutCode.Utf8: + return writer.WriteString(path, value.ToString()); + } + } + catch (OverflowException) + { + // Ignore overflow, and just write value as a long. + } + + return writer.WriteInt64(path, value); + } + + case JTokenType.Float: + { + double value = token.Value(); + try + { + switch (code) + { + case LayoutCode.Int8: + return writer.WriteInt8(path, (sbyte)value); + case LayoutCode.Int16: + return writer.WriteInt16(path, (short)value); + case LayoutCode.Int32: + return writer.WriteInt32(path, (int)value); + case LayoutCode.Int64: + return writer.WriteInt64(path, (long)value); + case LayoutCode.UInt8: + return writer.WriteUInt8(path, (byte)value); + case LayoutCode.UInt16: + return writer.WriteUInt16(path, (ushort)value); + case LayoutCode.UInt32: + return writer.WriteUInt32(path, (uint)value); + case LayoutCode.UInt64: + return writer.WriteUInt64(path, (ulong)value); + case LayoutCode.VarInt: + return writer.WriteVarInt(path, (long)value); + case LayoutCode.VarUInt: + return writer.WriteVarUInt(path, (ulong)value); + case LayoutCode.Float32: + return writer.WriteFloat32(path, (float)value); + case LayoutCode.Float64: + return writer.WriteFloat64(path, value); + case LayoutCode.Decimal: + return writer.WriteDecimal(path, (decimal)value); + case LayoutCode.Utf8: + return writer.WriteString(path, value.ToString(CultureInfo.InvariantCulture)); + } + } + catch (OverflowException) + { + // Ignore overflow, and just write value as a double. + } + + return writer.WriteFloat64(path, value); + } + + case JTokenType.String: + switch (code) + { + case LayoutCode.Boolean: + { + if (!bool.TryParse(token.Value(), out bool value)) + { + goto default; + } + + return writer.WriteBool(path, value); + } + + case LayoutCode.Int8: + { + if (!sbyte.TryParse(token.Value(), out sbyte value)) + { + goto default; + } + + return writer.WriteInt8(path, value); + } + + case LayoutCode.Int16: + { + if (!short.TryParse(token.Value(), out short value)) + { + goto default; + } + + return writer.WriteInt16(path, value); + } + + case LayoutCode.Int32: + { + if (!int.TryParse(token.Value(), out int value)) + { + goto default; + } + + return writer.WriteInt32(path, value); + } + + case LayoutCode.Int64: + { + if (!long.TryParse(token.Value(), out long value)) + { + goto default; + } + + return writer.WriteInt64(path, value); + } + + case LayoutCode.UInt8: + { + if (!byte.TryParse(token.Value(), out byte value)) + { + goto default; + } + + return writer.WriteUInt8(path, value); + } + + case LayoutCode.UInt16: + { + if (!ushort.TryParse(token.Value(), out ushort value)) + { + goto default; + } + + return writer.WriteUInt16(path, value); + } + + case LayoutCode.UInt32: + { + if (!uint.TryParse(token.Value(), out uint value)) + { + goto default; + } + + return writer.WriteUInt32(path, value); + } + + case LayoutCode.UInt64: + { + if (!ulong.TryParse(token.Value(), out ulong value)) + { + goto default; + } + + return writer.WriteUInt64(path, value); + } + + case LayoutCode.VarInt: + { + if (!long.TryParse(token.Value(), out long value)) + { + goto default; + } + + return writer.WriteVarInt(path, value); + } + + case LayoutCode.VarUInt: + { + if (!ulong.TryParse(token.Value(), out ulong value)) + { + goto default; + } + + return writer.WriteVarUInt(path, value); + } + + case LayoutCode.Float32: + { + if (!float.TryParse(token.Value(), out float value)) + { + goto default; + } + + return writer.WriteFloat32(path, value); + } + + case LayoutCode.Float64: + { + if (!double.TryParse(token.Value(), out double value)) + { + goto default; + } + + return writer.WriteFloat64(path, value); + } + + case LayoutCode.Decimal: + { + if (!decimal.TryParse(token.Value(), out decimal value)) + { + goto default; + } + + return writer.WriteDecimal(path, value); + } + + case LayoutCode.DateTime: + { + IFormatProvider provider = CultureInfo.CurrentCulture; + DateTimeStyles style = DateTimeStyles.AdjustToUniversal | + DateTimeStyles.AllowWhiteSpaces | + DateTimeStyles.AssumeUniversal; + + if (!DateTime.TryParse(token.Value(), provider, style, out DateTime value)) + { + goto default; + } + + return writer.WriteDateTime(path, value); + } + + case LayoutCode.Guid: + { + string s = token.Value(); + + // If the guid is quoted then remove the quotes. + if (s.Length > 2 && s[0] == '"' && s[s.Length - 1] == '"') + { + s = s.Substring(1, s.Length - 2); + } + + if (!Guid.TryParse(s, out Guid value)) + { + goto default; + } + + return writer.WriteGuid(path, value); + } + + case LayoutCode.Binary: + { + string s = token.Value(); + try + { + byte[] newBytes = Convert.FromBase64String(s); + return writer.WriteBinary(path, newBytes.AsSpan()); + } + catch (Exception) + { + // fall through and try hex. + } + + try + { + byte[] newBytes = ByteConverter.ToBytes(s); + return writer.WriteBinary(path, newBytes.AsSpan()); + } + catch (Exception) + { + goto default; + } + } + + case LayoutCode.UnixDateTime: + { + if (!long.TryParse(token.Value(), out long value)) + { + goto default; + } + + return writer.WriteUnixDateTime(path, new UnixDateTime(value)); + } + + case LayoutCode.MongoDbObjectId: + { + string s = token.Value(); + try + { + byte[] newBytes = Convert.FromBase64String(s); + return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(newBytes.AsSpan())); + } + catch (Exception) + { + // fall through and try hex. + } + + try + { + byte[] newBytes = ByteConverter.ToBytes(s); + return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(newBytes.AsSpan())); + } + catch (Exception) + { + goto default; + } + } + + default: + return writer.WriteString(path, token.Value()); + } + + case JTokenType.Boolean: + { + bool value = token.Value(); + switch (code) + { + case LayoutCode.Int8: + return writer.WriteInt8(path, (sbyte)(value ? 1 : 0)); + case LayoutCode.Int16: + return writer.WriteInt16(path, (short)(value ? 1 : 0)); + case LayoutCode.Int32: + return writer.WriteInt32(path, value ? 1 : 0); + case LayoutCode.Int64: + return writer.WriteInt64(path, value ? 1L : 0L); + case LayoutCode.UInt8: + return writer.WriteUInt8(path, (byte)(value ? 1 : 0)); + case LayoutCode.UInt16: + return writer.WriteUInt16(path, (ushort)(value ? 1 : 0)); + case LayoutCode.UInt32: + return writer.WriteUInt32(path, (uint)(value ? 1 : 0)); + case LayoutCode.UInt64: + return writer.WriteUInt64(path, value ? 1U : 0U); + case LayoutCode.VarInt: + return writer.WriteVarInt(path, value ? 1L : 0L); + case LayoutCode.VarUInt: + return writer.WriteVarUInt(path, value ? 1U : 0U); + case LayoutCode.Float32: + return writer.WriteFloat32(path, value ? 1 : 0); + case LayoutCode.Float64: + return writer.WriteFloat64(path, value ? 1 : 0); + case LayoutCode.Decimal: + return writer.WriteDecimal(path, value ? 1 : 0); + case LayoutCode.Utf8: + return writer.WriteString(path, value ? "true" : "false"); + default: + return writer.WriteBool(path, value); + } + } + + case JTokenType.Null: + switch (code) + { + case LayoutCode.Null: + case LayoutCode.Invalid: + return writer.WriteNull(path); + default: + + // Any other schematized type then just don't write it. + return Result.Success; + } + + case JTokenType.Comment: + return Result.Success; + default: + Console.Error.WriteLine($"Error: Unexpected JSON token type: {token.Type}."); + return Result.InvalidRow; + } + } + + private struct WriterContext + { + public JToken Token; + public string PathPrefix; + } + } +} diff --git a/dotnet/src/HybridRowCLI/Microsoft.Azure.Cosmos.Serialization.HybridRowCLI.csproj b/dotnet/src/HybridRowCLI/Microsoft.Azure.Cosmos.Serialization.HybridRowCLI.csproj new file mode 100644 index 0000000..859f8a7 --- /dev/null +++ b/dotnet/src/HybridRowCLI/Microsoft.Azure.Cosmos.Serialization.HybridRowCLI.csproj @@ -0,0 +1,30 @@ + + + + true + true + {F7D04E9B-4257-4D7E-9AAD-C743AEDBED04} + Exe + Test + Microsoft.Azure.Cosmos.Serialization.HybridRowCLI + Microsoft.Azure.Cosmos.Serialization.HybridRowCLI + netcoreapp2.2 + x64 + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/HybridRowCLI/PrintCommand.cs b/dotnet/src/HybridRowCLI/PrintCommand.cs new file mode 100644 index 0000000..12474df --- /dev/null +++ b/dotnet/src/HybridRowCLI/PrintCommand.cs @@ -0,0 +1,441 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Runtime.InteropServices; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Json; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.Extensions.CommandLineUtils; + + public class PrintCommand + { + private const int InitialCapacity = 2 * 1024 * 1024; + private bool showSchema; + + private bool verbose; + + private bool interactive; + + private bool outputJson; + + private char outputJsonQuote; + + private string outputJsonIndent; + + private string namespaceFile; + + private List rows; + + private PrintCommand() + { + } + + public static void AddCommand(CommandLineApplication app) + { + app.Command( + "print", + command => + { + command.Description = "Print hybrid row value(s)."; + command.ExtendedHelpText = + "Convert binary serialized hybrid row value(s) to a printable string form using the given schema.\n\n" + + "The print command accepts files in three formats:\n" + + "\t* A HybridRow RecordIO file containing 0 or more records (with or without embedded SDL).\n" + + "\t* A HybridRow binary file containing exactly 1 record. The length of the file indicates the record size.\n" + + "\t* A HybridRow text file containing exactly 1 record written as a HEX text string. The length of the file\n" + + "\t indicates the length of the encoding. All non-HEX characters (e.g. extra spaces, newlines, or dashes)\n" + + "\t are ignored when decoding the file from HEX string to binary."; + + command.HelpOption("-? | -h | --help"); + + CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue); + CommandOption showSchemaOpt = command.Option( + "--show-schema", + "Include embedded the schema in the output.", + CommandOptionType.NoValue); + + CommandOption interactiveOpt = command.Option("-i|--interactive", "Use interactive interface.", CommandOptionType.NoValue); + CommandOption jsonOpt = command.Option("-j|--json", "Output in JSON.", CommandOptionType.NoValue); + CommandOption jsonSingleQuoteOpt = command.Option("--json-single-quote", "Use single quotes in JSON.", CommandOptionType.NoValue); + CommandOption jsonNoIndentOpt = command.Option("--json-no-indent", "Don't indent in JSON.", CommandOptionType.NoValue); + + CommandOption namespaceOpt = command.Option( + "-n|--namespace", + "File containing the schema namespace.", + CommandOptionType.SingleValue); + + CommandArgument rowsOpt = command.Argument( + "rows", + "File(s) containing rows to print.", + arg => { arg.MultipleValues = true; }); + + command.OnExecute( + () => + { + PrintCommand config = new PrintCommand + { + verbose = verboseOpt.HasValue(), + showSchema = showSchemaOpt.HasValue(), + outputJson = jsonOpt.HasValue(), + outputJsonQuote = jsonSingleQuoteOpt.HasValue() ? '\'' : '"', + outputJsonIndent = jsonNoIndentOpt.HasValue() ? null : " ", + interactive = interactiveOpt.HasValue(), + namespaceFile = namespaceOpt.Value(), + rows = rowsOpt.Values + }; + + return config.OnExecuteAsync().Result; + }); + }); + } + + /// + /// Read a from the current stream position without moving the + /// position. + /// + /// The stream to read from. + /// The header read at the current position. + /// + /// If a header cannot be read from the current stream position for any + /// reason. + /// + /// + /// On success the stream's position is not changed. On error the stream position is + /// undefined. + /// + private static async Task PeekHybridRowHeaderAsync(Stream stm) + { + try + { + Memory bytes = await PrintCommand.ReadFixedAsync(stm, HybridRowHeader.Size); + return MemoryMarshal.Read(bytes.Span); + } + finally + { + stm.Seek(-HybridRowHeader.Size, SeekOrigin.Current); + } + } + + /// Reads a fixed length segment from a stream. + /// The stream to read from. + /// The length to read in bytes. + /// A sequence of bytes read from the stream exactly long. + /// + /// if bytes cannot be read from the current + /// stream position. + /// + private static async Task> ReadFixedAsync(Stream stm, int length) + { + Memory bytes = new byte[length].AsMemory(); + Memory active = bytes; + int bytesRead; + do + { + bytesRead = await stm.ReadAsync(active); + active = active.Slice(bytesRead); + } + while (bytesRead != 0); + + if (active.Length != 0) + { + throw new Exception("Failed to parse row header"); + } + + return bytes; + } + + private static void Refresh(ConsoleNative.ConsoleModes origMode, Layout layout, int length, long index, string str) + { + string[] lines = str.Split('\n'); + int height = lines.Length + 2; + int width = (from line in lines select line.Length).Max(); + height = Math.Max(height, Console.WindowHeight); + width = Math.Max(width, Console.WindowWidth); + ConsoleNative.Mode = origMode & ~ConsoleNative.ConsoleModes.ENABLE_WRAP_AT_EOL_OUTPUT; + Console.CursorVisible = false; + ConsoleNative.SetBufferSize(width, height); + + Console.Clear(); + if (layout != null) + { + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine($"[{index}] Schema: {layout.SchemaId} {layout.Name}, Length: {length}"); + } + + Console.ForegroundColor = ConsoleColor.Gray; + + Console.WriteLine(str); + Console.SetWindowPosition(0, 0); + } + + private static bool ShowInteractive(Layout layout, int length, long index, string str) + { + int origBufferHeight = Console.BufferHeight; + int origBufferWidth = Console.BufferWidth; + ConsoleNative.ConsoleModes origMode = ConsoleNative.Mode; + try + { + PrintCommand.Refresh(origMode, layout, length, index, str); + + while (true) + { + ConsoleKeyInfo cki = Console.ReadKey(true); + if ((cki.Modifiers & ConsoleModifiers.Alt) != 0 || + (cki.Modifiers & ConsoleModifiers.Shift) != 0 || + (cki.Modifiers & ConsoleModifiers.Control) != 0) + { + continue; + } + + int top; + int left; + switch (cki.Key) + { + case ConsoleKey.Q: + case ConsoleKey.Escape: + return false; + + case ConsoleKey.Spacebar: + case ConsoleKey.N: + case ConsoleKey.Enter: + return true; + + case ConsoleKey.R: + PrintCommand.Refresh(origMode, layout, length, index, str); + break; + + case ConsoleKey.Home: + Console.SetWindowPosition(0, 0); + break; + + case ConsoleKey.End: + Console.SetWindowPosition(Console.WindowLeft, Console.BufferHeight - Console.WindowHeight); + break; + + case ConsoleKey.PageDown: + top = Console.WindowTop + Console.WindowHeight; + top = Math.Min(top, Console.BufferHeight - Console.WindowHeight); + Console.SetWindowPosition(Console.WindowLeft, top); + break; + + case ConsoleKey.PageUp: + top = Console.WindowTop - Console.WindowHeight; + top = Math.Max(top, 0); + Console.SetWindowPosition(Console.WindowLeft, top); + break; + + case ConsoleKey.DownArrow: + top = Console.WindowTop + 1; + top = Math.Min(top, Console.BufferHeight - Console.WindowHeight); + Console.SetWindowPosition(Console.WindowLeft, top); + break; + + case ConsoleKey.UpArrow: + top = Console.WindowTop - 1; + top = Math.Max(top, 0); + Console.SetWindowPosition(Console.WindowLeft, top); + break; + + case ConsoleKey.RightArrow: + left = Console.WindowLeft + 1; + left = Math.Min(left, Console.BufferWidth - Console.WindowWidth); + Console.SetWindowPosition(left, Console.WindowTop); + break; + + case ConsoleKey.LeftArrow: + left = Console.WindowLeft - 1; + left = Math.Max(left, 0); + Console.SetWindowPosition(left, Console.WindowTop); + break; + } + } + } + finally + { + ConsoleNative.Mode = origMode; + Console.ResetColor(); + origBufferWidth = Math.Max(Console.WindowWidth, origBufferWidth); + origBufferHeight = Math.Max(Console.WindowHeight, origBufferHeight); + Console.SetBufferSize(origBufferWidth, origBufferHeight); + Console.Clear(); + Console.CursorVisible = true; + } + } + + /// Convert a text stream containing a sequence of HEX characters into a byte sequence. + /// The stream to read the text from. + /// The entire contents of the stream interpreted as HEX characters. + /// Any character that is not a HEX character is ignored. + private static async Task> HexTextStreamToBinaryAsync(Stream stm) + { + using (TextReader reader = new StreamReader(stm)) + { + string rowAsText = await reader.ReadToEndAsync(); + string trimmed = string.Concat( + from c in rowAsText + where (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') + select c); + + return ByteConverter.ToBytes(trimmed).AsMemory(); + } + } + + private async Task OnExecuteAsync() + { + LayoutResolver globalResolver = await SchemaUtil.CreateResolverAsync(this.namespaceFile, this.verbose); + MemorySpanResizer resizer = new MemorySpanResizer(PrintCommand.InitialCapacity); + foreach (string rowFile in this.rows) + { + using (Stream stm = new FileStream(rowFile, FileMode.Open)) + { + // Detect if it is a text or binary file via the encoding of the magic number at the + // beginning of the HybridRow header. + int magicNumber = stm.ReadByte(); + stm.Seek(-1, SeekOrigin.Current); + if (magicNumber == -1) + { + continue; // empty file + } + + Result r = Result.Failure; + if (magicNumber == (int)HybridRowVersion.V1) + { + HybridRowHeader header = await PrintCommand.PeekHybridRowHeaderAsync(stm); + if (header.SchemaId == SystemSchema.SegmentSchemaId) + { + r = await this.PrintRecordIOAsync(stm, globalResolver, resizer); + } + else + { + Memory rowAsBinary = await PrintCommand.ReadFixedAsync(stm, (int)stm.Length); + r = this.PrintOneRow(rowAsBinary, 0, globalResolver); + } + } + else if (char.ToUpper((char)magicNumber, CultureInfo.InvariantCulture) == HybridRowVersion.V1.ToString("X")[0]) + { + // Convert hex text file to binary. + Memory rowAsBinary = await PrintCommand.HexTextStreamToBinaryAsync(stm); + r = this.PrintOneRow(rowAsBinary, 0, globalResolver); + } + + if (r == Result.Canceled) + { + return 0; + } + + if (r != Result.Success) + { + await Console.Error.WriteLineAsync($"Error reading row at {rowFile}"); + return -1; + } + } + } + + return 0; + } + + /// Print all records from a HybridRow RecordIO file. + /// Stream containing the HybridRow RecordIO content. + /// The global resolver to use when segments don't contain embedded SDL. + /// The resizer for allocating row buffers. + /// Success if the print is successful, an error code otherwise. + private async Task PrintRecordIOAsync(Stream stm, LayoutResolver globalResolver, MemorySpanResizer resizer) + { + LayoutResolver segmentResolver = globalResolver; + + // Read a RecordIO stream. + long index = 0; + Result r = await stm.ReadRecordIOAsync( + record => this.PrintOneRow(record, index++, segmentResolver), + segment => + { + r = SegmentSerializer.Read(segment.Span, globalResolver, out Segment s); + if (r != Result.Success) + { + return r; + } + + segmentResolver = string.IsNullOrWhiteSpace(s.SDL) ? globalResolver : SchemaUtil.LoadFromSdl(s.SDL, this.verbose, globalResolver); + + if (this.showSchema) + { + string str = string.IsNullOrWhiteSpace(s.SDL) ? "" : s.SDL; + if (!this.interactive) + { + Console.WriteLine(str); + } + else + { + if (!PrintCommand.ShowInteractive(null, 0, -1, str)) + { + return Result.Canceled; + } + } + } + + return Result.Success; + }, + resizer); + + return r; + } + + /// Print a single HybridRow record. + /// The raw bytes of the row. + /// + /// A 0-based index of the row relative to its outer container (for display + /// purposes only). + /// + /// The resolver for nested contents within the row. + /// Success if the print is successful, an error code otherwise. + private Result PrintOneRow(Memory buffer, long index, LayoutResolver resolver) + { + RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, resolver); + RowReader reader = new RowReader(ref row); + Result r; + string str; + if (this.outputJson) + { + r = reader.ToJson(new RowReaderJsonSettings(this.outputJsonIndent, this.outputJsonQuote), out str); + } + else + { + r = DiagnosticConverter.ReaderToString(ref reader, out str); + } + + if (r != Result.Success) + { + return r; + } + + Layout layout = resolver.Resolve(row.Header.SchemaId); + if (!this.interactive) + { + Console.WriteLine($"Schema: {layout.SchemaId} {layout.Name}, Length: {row.Length}"); + Console.WriteLine(str); + } + else + { + if (!PrintCommand.ShowInteractive(layout, row.Length, index, str)) + { + return Result.Canceled; + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRowCLI/Properties/AssemblyInfo.cs b/dotnet/src/HybridRowCLI/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..4cfce31 --- /dev/null +++ b/dotnet/src/HybridRowCLI/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRowCLI")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("F7D04E9B-4257-4D7E-9AAD-C743AEDBED04")] diff --git a/dotnet/src/HybridRowCLI/SchemaUtil.cs b/dotnet/src/HybridRowCLI/SchemaUtil.cs new file mode 100644 index 0000000..4db4b63 --- /dev/null +++ b/dotnet/src/HybridRowCLI/SchemaUtil.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public static class SchemaUtil + { + /// Create a resolver. + /// + /// Optional namespace file containing a namespace to be included in the + /// resolver. + /// + /// True if verbose output should be written to stdout. + /// A resolver. + public static async Task CreateResolverAsync(string namespaceFile, bool verbose) + { + LayoutResolver globalResolver; + if (string.IsNullOrWhiteSpace(namespaceFile)) + { + globalResolver = SystemSchema.LayoutResolver; + } + else + { + if (verbose) + { + Console.WriteLine($"Loading {namespaceFile}..."); + Console.WriteLine(); + } + + string json = await File.ReadAllTextAsync(namespaceFile); + globalResolver = SchemaUtil.LoadFromSdl(json, verbose, SystemSchema.LayoutResolver); + } + + Contract.Requires(globalResolver != null); + return globalResolver; + } + + /// Create a HybridRow resolver for given piece of embedded Schema Definition Language (SDL). + /// The SDL to parse. + /// True if verbose output should be written to stdout. + /// A resolver that resolves all types in the given SDL. + public static LayoutResolver LoadFromSdl(string json, bool verbose, LayoutResolver parent = default) + { + Namespace n = Namespace.Parse(json); + if (verbose) + { + Console.WriteLine($"Namespace: {n.Name}"); + foreach (Schema s in n.Schemas) + { + Console.WriteLine($" {s.SchemaId} Schema: {s.Name}"); + } + } + + LayoutResolver resolver = new LayoutResolverNamespace(n, parent); + if (verbose) + { + Console.WriteLine(); + Console.WriteLine($"Loaded {n.Name}.\n"); + } + + return resolver; + } + } +} diff --git a/dotnet/src/HybridRowCLI/StringExtensions.cs b/dotnet/src/HybridRowCLI/StringExtensions.cs new file mode 100644 index 0000000..89ac764 --- /dev/null +++ b/dotnet/src/HybridRowCLI/StringExtensions.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI +{ + using System; + + // TODO: this class should go away once we move to .NET Core 2.1. + internal static class StringExtensions + { + public static unsafe string AsString(this ReadOnlySpan span) + { + fixed (char* p = span) + { + return new string(p, 0, span.Length); + } + } + } +} diff --git a/dotnet/src/HybridRowGenerator/ByteConverter.cs b/dotnet/src/HybridRowGenerator/ByteConverter.cs new file mode 100644 index 0000000..a334caf --- /dev/null +++ b/dotnet/src/HybridRowGenerator/ByteConverter.cs @@ -0,0 +1,114 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using Microsoft.Azure.Cosmos.Core; + + public static unsafe class ByteConverter + { + private static readonly byte[] DecodeTable = + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + }; + + private static readonly uint[] EncodeTable = ByteConverter.Initialize(); + + public static string ToHex(ReadOnlySpan bytes) + { + int len = bytes.Length; + string result = new string((char)0, len * 2); + fixed (uint* lp = ByteConverter.EncodeTable) + fixed (byte* bp = bytes) + fixed (char* rp = result) + { + for (int i = 0; i < len; i++) + { + ((uint*)rp)[i] = lp[bp[i]]; + } + } + + return result; + } + + public static byte[] ToBytes(string hexChars) + { + Contract.Requires(hexChars != null); + Contract.Requires(hexChars.Length % 2 == 0); + + int len = hexChars.Length; + byte[] result = new byte[len / 2]; + fixed (byte* lp = ByteConverter.DecodeTable) + fixed (char* cp = hexChars) + fixed (byte* rp = result) + { + for (int i = 0; i < len; i += 2) + { + int c1 = cp[i]; + if ((c1 < 0) || (c1 > 255)) + { + throw new Exception($"Invalid character: {c1}"); + } + + byte b1 = lp[c1]; + if (b1 == 255) + { + throw new Exception($"Invalid character: {c1}"); + } + + int c2 = cp[i + 1]; + if ((c2 < 0) || (c2 > 255)) + { + throw new Exception($"Invalid character: {c2}"); + } + + byte b2 = lp[c2]; + if (b2 == 255) + { + throw new Exception($"Invalid character: {c2}"); + } + + rp[i / 2] = (byte)((b1 << 4) | b2); + } + } + + return result; + } + + private static uint[] Initialize() + { + uint[] result = new uint[256]; + for (int i = 0; i < 256; i++) + { + string s = i.ToString("X2"); + if (BitConverter.IsLittleEndian) + { + result[i] = ((uint)s[0]) + ((uint)s[1] << 16); + } + else + { + result[i] = ((uint)s[1]) + ((uint)s[0] << 16); + } + } + + return result; + } + } +} diff --git a/dotnet/src/HybridRowGenerator/CharDistribution.cs b/dotnet/src/HybridRowGenerator/CharDistribution.cs new file mode 100644 index 0000000..3ea436b --- /dev/null +++ b/dotnet/src/HybridRowGenerator/CharDistribution.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + + public class CharDistribution + { + private readonly char min; + private readonly char max; + private readonly DistributionType type; + + public CharDistribution(char min, char max, DistributionType type = DistributionType.Uniform) + { + this.min = min; + this.max = max; + this.type = type; + } + + public char Min => this.min; + + public char Max => this.max; + + public DistributionType Type => this.type; + + public char Next(RandomGenerator rand) + { + Contract.Requires(this.type == DistributionType.Uniform); + + return (char)rand.NextUInt16(this.min, this.max); + } + } +} diff --git a/dotnet/src/HybridRowGenerator/DiagnosticConverter.cs b/dotnet/src/HybridRowGenerator/DiagnosticConverter.cs new file mode 100644 index 0000000..73e849b --- /dev/null +++ b/dotnet/src/HybridRowGenerator/DiagnosticConverter.cs @@ -0,0 +1,749 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using System.Collections.Generic; + using System.Text; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public static class DiagnosticConverter + { + public static Result ReaderToString(ref RowReader reader, out string str) + { + ReaderStringContext ctx = new ReaderStringContext(new StringBuilder()); + Result result = DiagnosticConverter.ReaderToString(ref reader, ctx); + if (result != Result.Success) + { + str = null; + return result; + } + + str = ctx.Builder.ToString(); + return Result.Success; + } + + public static Result ReaderToDynamic(ref RowReader reader, out Dictionary scope) + { + scope = new Dictionary(SamplingUtf8StringComparer.Default); + return DiagnosticConverter.ReaderToDynamic(ref reader, scope); + } + + private static Result ReaderToDynamic(ref RowReader reader, object scope) + { + while (reader.Read()) + { + Result r; + switch (reader.Type.LayoutCode) + { + case LayoutCode.Null: + { + r = reader.ReadNull(out NullValue value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Boolean: + { + r = reader.ReadBool(out bool value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Int8: + { + r = reader.ReadInt8(out sbyte value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Int16: + { + r = reader.ReadInt16(out short value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Int32: + { + r = reader.ReadInt32(out int value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Int64: + { + r = reader.ReadInt64(out long value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.UInt8: + { + r = reader.ReadUInt8(out byte value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.UInt16: + { + r = reader.ReadUInt16(out ushort value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.UInt32: + { + r = reader.ReadUInt32(out uint value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.UInt64: + { + r = reader.ReadUInt64(out ulong value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.VarInt: + { + r = reader.ReadVarInt(out long value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.VarUInt: + { + r = reader.ReadVarUInt(out ulong value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Float32: + { + r = reader.ReadFloat32(out float value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Float64: + { + r = reader.ReadFloat64(out double value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Float128: + { + r = reader.ReadFloat128(out Float128 value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Decimal: + { + r = reader.ReadDecimal(out decimal value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.DateTime: + { + r = reader.ReadDateTime(out DateTime value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.UnixDateTime: + { + r = reader.ReadUnixDateTime(out UnixDateTime value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Guid: + { + r = reader.ReadGuid(out Guid value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.MongoDbObjectId: + { + r = reader.ReadMongoDbObjectId(out MongoDbObjectId value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Utf8: + { + r = reader.ReadString(out Utf8String value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.Binary: + { + r = reader.ReadBinary(out byte[] value); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, value); + break; + } + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + { + if (!reader.HasValue) + { + break; + } + + goto case LayoutCode.TypedTupleScope; + } + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + { + object childScope = new Dictionary(SamplingUtf8StringComparer.Default); + r = reader.ReadScope(childScope, DiagnosticConverter.ReaderToDynamic); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, childScope); + break; + } + + case LayoutCode.ArrayScope: + case LayoutCode.ImmutableArrayScope: + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + { + object childScope = new List(); + r = reader.ReadScope(childScope, DiagnosticConverter.ReaderToDynamic); + if (r != Result.Success) + { + return r; + } + + DiagnosticConverter.AddToScope(scope, reader.Path, childScope); + break; + } + + default: + { + Contract.Assert(false, $"Unknown type will be ignored: {reader.Type.LayoutCode}"); + break; + } + } + } + + return Result.Success; + } + + private static void AddToScope(object scope, UtfAnyString path, object value) + { + if (scope is List linearScope) + { + Contract.Assert(path.IsNull); + linearScope.Add(value); + } + else + { + Contract.Assert(!string.IsNullOrWhiteSpace(path.ToString())); + ((Dictionary)scope)[path.ToUtf8String()] = value; + } + } + + private static Result ReaderToString(ref RowReader reader, ReaderStringContext ctx) + { + while (reader.Read()) + { + string path = !reader.Path.IsNull ? $"\"{reader.Path}\"" : "null"; + ctx.Builder.Append( + $"{new string(' ', ctx.Indent * 2)}Storage: {reader.Storage}, Path: {path}, Index: {reader.Index}, Type: {reader.Type.Name + reader.TypeArgs.ToString()}, Value: "); + + Result r; + switch (reader.Type.LayoutCode) + { + case LayoutCode.Null: + { + r = reader.ReadNull(out NullValue _); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.AppendLine("null"); + break; + } + + case LayoutCode.Boolean: + case LayoutCode.BooleanFalse: + { + r = reader.ReadBool(out bool value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Int8: + { + r = reader.ReadInt8(out sbyte value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Int16: + { + r = reader.ReadInt16(out short value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Int32: + { + r = reader.ReadInt32(out int value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Int64: + { + r = reader.ReadInt64(out long value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.UInt8: + { + r = reader.ReadUInt8(out byte value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.UInt16: + { + r = reader.ReadUInt16(out ushort value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.UInt32: + { + r = reader.ReadUInt32(out uint value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.UInt64: + { + r = reader.ReadUInt64(out ulong value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.VarInt: + { + r = reader.ReadVarInt(out long value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.VarUInt: + { + r = reader.ReadVarUInt(out ulong value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Float32: + { + r = reader.ReadFloat32(out float value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Float64: + { + r = reader.ReadFloat64(out double value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Float128: + { + r = reader.ReadFloat128(out Float128 value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.AppendFormat("High: {0}, Low: {1}\n", value.High, value.Low); + break; + } + + case LayoutCode.Decimal: + { + r = reader.ReadDecimal(out decimal value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.DateTime: + { + r = reader.ReadDateTime(out DateTime value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.UnixDateTime: + { + r = reader.ReadUnixDateTime(out UnixDateTime value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.Append(value.Milliseconds); + ctx.Builder.AppendLine(); + break; + } + + case LayoutCode.Guid: + { + r = reader.ReadGuid(out Guid value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.AppendLine(value.ToString()); + break; + } + + case LayoutCode.MongoDbObjectId: + { + r = reader.ReadMongoDbObjectId(out MongoDbObjectId value); + if (r != Result.Success) + { + return r; + } + + ReadOnlyMemory bytes = value.ToByteArray(); + ctx.Builder.AppendLine(ByteConverter.ToHex(bytes.Span)); + break; + } + + case LayoutCode.Utf8: + { + r = reader.ReadString(out Utf8String value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.AppendLine(value.ToString()); + break; + } + + case LayoutCode.Binary: + { + r = reader.ReadBinary(out ReadOnlySpan value); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.AppendLine(ByteConverter.ToHex(value)); + break; + } + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + { + if (!reader.HasValue) + { + ctx.Builder.AppendLine("null"); + break; + } + + goto case LayoutCode.TypedTupleScope; + } + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + case LayoutCode.ArrayScope: + case LayoutCode.ImmutableArrayScope: + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + { + ctx.Builder.AppendLine("{"); + r = reader.ReadScope(new ReaderStringContext(ctx.Builder, ctx.Indent + 1), DiagnosticConverter.ReaderToString); + if (r != Result.Success) + { + return r; + } + + ctx.Builder.AppendLine($"{new string(' ', ctx.Indent * 2)}}}"); + break; + } + + default: + { + Contract.Assert(false, $"Unknown type will be ignored: {reader.Type.LayoutCode}"); + break; + } + } + } + + return Result.Success; + } + + private readonly struct ReaderStringContext + { + public readonly int Indent; + public readonly StringBuilder Builder; + + public ReaderStringContext(StringBuilder builder, int indent = 0) + { + this.Indent = indent; + this.Builder = builder; + } + } + } +} diff --git a/dotnet/src/HybridRowGenerator/DistributionType.cs b/dotnet/src/HybridRowGenerator/DistributionType.cs new file mode 100644 index 0000000..e8e6ae0 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/DistributionType.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + public enum DistributionType + { + Uniform = 0, + } +} diff --git a/dotnet/src/HybridRowGenerator/HybridRowGeneratorConfig.cs b/dotnet/src/HybridRowGenerator/HybridRowGeneratorConfig.cs new file mode 100644 index 0000000..bd100e3 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/HybridRowGeneratorConfig.cs @@ -0,0 +1,105 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public class HybridRowGeneratorConfig + { + /// The number of attempts to allocate a value given an exclusion list before aborting. + private const int ConflictRetryAttemptsDefault = 100; + + /// + /// The rate at which the cardinality of substructures decays as a function of depth. This + /// ensures that randomly generated values don't become infinitely large. + /// + private const double DepthDecayFactorDefault = -1.0D; + + /// + /// The length (in chars) of identifiers including namespace names, schema names, property + /// names, etc. + /// + private static readonly IntDistribution IdentifierLengthDefault = new IntDistribution(1, 20); + + /// + /// The distribution of unicode characters used in constructing identifiers including + /// namespace names, schema names, property names, etc. + /// + private static readonly CharDistribution IdentifierCharactersDefault = new CharDistribution('a', 'z'); + + /// The length (in chars) of annotation comments within the schema. + private static readonly IntDistribution CommentLengthDefault = new IntDistribution(0, 50); + + /// The length (in chars) of string values. + private static readonly IntDistribution StringValueLengthDefault = new IntDistribution(0, 100); + + /// The length (in bytes) of binary values. + private static readonly IntDistribution BinaryValueLengthDefault = new IntDistribution(0, 100); + + /// The length (in number of elements) of collection scopes. + private static readonly IntDistribution CollectionValueLengthDefault = new IntDistribution(0, 10); + + /// The distribution of unicode characters used in constructing Unicode field values. + private static readonly CharDistribution UnicodeCharactersDefault = new CharDistribution('\u0001', char.MaxValue); + + /// The space of SchemaId values assigned to schemas (table or UDT) within a single namespace. + private static readonly IntDistribution SchemaIdsDefault = new IntDistribution(int.MinValue, int.MaxValue); + + /// The number of properties (i.e. columns, fields) to appear in a table or UDT definition. + private static readonly IntDistribution NumTablePropertiesDefault = new IntDistribution(1, 10); + + /// The number of items to appear in a tuple field. + private static readonly IntDistribution NumTupleItemsDefault = new IntDistribution(2, 5); + + /// The number of items to appear in a tagged field. + private static readonly IntDistribution NumTaggedItemsDefault = new IntDistribution(1, 2); + + /// The length (in units, e.g. chars, bytes, etc.) of variable length primitive field values. + private static readonly IntDistribution PrimitiveFieldValueLengthDefault = new IntDistribution(1, 1024); + + /// The distribution of types for fields. + private static readonly IntDistribution FieldTypeDefault = new IntDistribution((int)TypeKind.Null, (int)TypeKind.Schema); + + /// The distribution of storage for fields. + private static readonly IntDistribution FieldStorageDefault = new IntDistribution((int)StorageKind.Sparse, (int)StorageKind.Variable); + + /// The distribution of initial sizes for RowBuffers. + private static readonly IntDistribution RowBufferInitialCapacityDefault = new IntDistribution(0, 2 * 1024 * 1024); + + public IntDistribution IdentifierLength { get; set; } = HybridRowGeneratorConfig.IdentifierLengthDefault; + + public CharDistribution IdentifierCharacters { get; set; } = HybridRowGeneratorConfig.IdentifierCharactersDefault; + + public IntDistribution CommentLength { get; set; } = HybridRowGeneratorConfig.CommentLengthDefault; + + public IntDistribution StringValueLength { get; set; } = HybridRowGeneratorConfig.StringValueLengthDefault; + + public IntDistribution BinaryValueLength { get; set; } = HybridRowGeneratorConfig.BinaryValueLengthDefault; + + public IntDistribution CollectionValueLength { get; set; } = HybridRowGeneratorConfig.CollectionValueLengthDefault; + + public CharDistribution UnicodeCharacters { get; set; } = HybridRowGeneratorConfig.UnicodeCharactersDefault; + + public IntDistribution SchemaIds { get; set; } = HybridRowGeneratorConfig.SchemaIdsDefault; + + public IntDistribution NumTableProperties { get; set; } = HybridRowGeneratorConfig.NumTablePropertiesDefault; + + public IntDistribution NumTupleItems { get; set; } = HybridRowGeneratorConfig.NumTupleItemsDefault; + + public IntDistribution NumTaggedItems { get; set; } = HybridRowGeneratorConfig.NumTaggedItemsDefault; + + public IntDistribution PrimitiveFieldValueLength { get; set; } = HybridRowGeneratorConfig.PrimitiveFieldValueLengthDefault; + + public IntDistribution FieldType { get; set; } = HybridRowGeneratorConfig.FieldTypeDefault; + + public IntDistribution FieldStorage { get; set; } = HybridRowGeneratorConfig.FieldStorageDefault; + + public IntDistribution RowBufferInitialCapacity { get; set; } = HybridRowGeneratorConfig.RowBufferInitialCapacityDefault; + + public int ConflictRetryAttempts { get; set; } = HybridRowGeneratorConfig.ConflictRetryAttemptsDefault; + + public double DepthDecayFactor { get; set; } = HybridRowGeneratorConfig.DepthDecayFactorDefault; + } +} diff --git a/dotnet/src/HybridRowGenerator/HybridRowValueGenerator.cs b/dotnet/src/HybridRowGenerator/HybridRowValueGenerator.cs new file mode 100644 index 0000000..a3447b9 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/HybridRowValueGenerator.cs @@ -0,0 +1,635 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#pragma warning disable SA1137 // Elements should have the same indentation + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public class HybridRowValueGenerator + { + private readonly RandomGenerator rand; + private readonly HybridRowGeneratorConfig config; + + public HybridRowValueGenerator(RandomGenerator rand, HybridRowGeneratorConfig config) + { + this.rand = rand; + this.config = config; + } + + public static bool DynamicTypeArgumentEquals(LayoutResolver resolver, object left, object right, TypeArgument typeArg) + { + if (object.ReferenceEquals(left, null)) + { + return object.ReferenceEquals(null, right); + } + + if (object.ReferenceEquals(null, right)) + { + return false; + } + + switch (typeArg.Type.LayoutCode) + { + case LayoutCode.Null: + case LayoutCode.Boolean: + case LayoutCode.Int8: + case LayoutCode.Int16: + case LayoutCode.Int32: + case LayoutCode.Int64: + case LayoutCode.UInt8: + case LayoutCode.UInt16: + case LayoutCode.UInt32: + case LayoutCode.UInt64: + case LayoutCode.VarInt: + case LayoutCode.VarUInt: + case LayoutCode.Float32: + case LayoutCode.Float64: + case LayoutCode.Float128: + case LayoutCode.Decimal: + case LayoutCode.DateTime: + case LayoutCode.UnixDateTime: + case LayoutCode.Guid: + case LayoutCode.MongoDbObjectId: + case LayoutCode.Utf8: + return object.Equals(left, right); + case LayoutCode.Binary: + return ((byte[])left).SequenceEqual((byte[])right); + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + { + Dictionary leftDict = (Dictionary)left; + Dictionary rightDict = (Dictionary)right; + if (leftDict.Count != rightDict.Count) + { + return false; + } + + // TODO: add properties to an object scope. + return true; + } + + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + { + List leftList = (List)left; + List rightList = (List)right; + if (leftList.Count != rightList.Count) + { + return false; + } + + for (int i = 0; i < leftList.Count; i++) + { + if (!HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, leftList[i], rightList[i], typeArg.TypeArgs[0])) + { + return false; + } + } + + return true; + } + + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + { + List leftList = (List)left; + List rightList = (List)right; + if (leftList.Count != rightList.Count) + { + return false; + } + + List working = new List(leftList); + foreach (object rightItem in rightList) + { + int i = HybridRowValueGenerator.SetContains(resolver, working, rightItem, typeArg); + if (i == -1) + { + return false; + } + + working.RemoveAt(i); + } + + return true; + } + + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + { + List> leftList = (List>)left; + List> rightList = (List>)right; + if (leftList.Count != rightList.Count) + { + return false; + } + + List> working = new List>(leftList); + foreach (List rightItem in rightList) + { + int i = HybridRowValueGenerator.MapContains(resolver, working, rightItem[0], typeArg); + if (i == -1) + { + return false; + } + + List leftItem = working[i]; + if (!HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, leftItem[1], rightItem[1], typeArg.TypeArgs[1])) + { + return false; + } + + working.RemoveAt(i); + } + + return true; + } + + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + { + List leftList = (List)left; + List rightList = (List)right; + if (leftList.Count != rightList.Count) + { + return false; + } + + Contract.Assert(leftList.Count == typeArg.TypeArgs.Count); + for (int i = 0; i < leftList.Count; i++) + { + if (!HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, leftList[i], rightList[i], typeArg.TypeArgs[i])) + { + return false; + } + } + + return true; + } + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + { + Contract.Assert(typeArg.TypeArgs.Count == 1); + return HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, left, right, typeArg.TypeArgs[0]); + } + + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + { + SchemaId schemaId = typeArg.TypeArgs.SchemaId; + Contract.Assert(schemaId != SchemaId.Invalid); + Layout layout = resolver.Resolve(schemaId); + Contract.Assert(layout != null); + + Dictionary leftDict = (Dictionary)left; + Dictionary rightDict = (Dictionary)right; + if (leftDict.Count != rightDict.Count) + { + return false; + } + + foreach (KeyValuePair pair in leftDict) + { + if (!rightDict.TryGetValue(pair.Key, out object rightValue)) + { + return false; + } + + Contract.Requires(layout.TryFind(pair.Key, out LayoutColumn c)); + return HybridRowValueGenerator.DynamicTypeArgumentEquals( + resolver, + pair.Value, + rightValue, + new TypeArgument(c.Type, c.TypeArgs)); + } + + return true; + } + + default: + Contract.Assert(false, $"Unknown type will be ignored: {typeArg}"); + return false; + } + } + + public static T GenerateExclusive(Func op, IEnumerable exclusions, int maxCandidates) + { + HashSet exclusionSet = new HashSet(exclusions); + T candidate; + + int retryCount = 0; + do + { + if (retryCount >= maxCandidates) + { + throw new Exception( + $"Max Candidates Reached: {maxCandidates} : " + + string.Join(",", from e in exclusions select e.ToString())); + } + + candidate = op(); + retryCount++; + } + while (exclusionSet.Contains(candidate)); + + return candidate; + } + + public string GenerateIdentifier() + { + int length = this.config.IdentifierLength.Next(this.rand); + CharDistribution distribution = this.config.IdentifierCharacters; + return this.GenerateString(length, distribution); + } + + public unsafe string GenerateString(int length, CharDistribution distribution) + { + if (length == 0) + { + return string.Empty; + } + + char* result = stackalloc char[length]; + int trim; + do + { + for (int i = 0; i < length; i++) + { + result[i] = distribution.Next(this.rand); + } + + // Drop characters until we are under the encoded length. + trim = length; + while ((trim > 0) && Encoding.UTF8.GetByteCount(result, trim) > length) + { + trim--; + result[trim] = '\0'; + } + } + while (trim == 0); + + // Pad with zero's if the resulting encoding is too short. + while (Encoding.UTF8.GetByteCount(result, trim) < length) + { + trim++; + } + + Contract.Assert(Encoding.UTF8.GetByteCount(result, trim) == length); + return new string(result, 0, trim); + } + + public byte[] GenerateBinary(int length) + { + byte[] result = new byte[length]; + for (int i = 0; i < length; i++) + { + result[i] = this.rand.NextUInt8(); + } + + return result; + } + + public SchemaId GenerateSchemaId() + { + return new SchemaId(this.config.SchemaIds.Next(this.rand)); + } + + public StorageKind GenerateStorageKind() + { + return (StorageKind)this.config.FieldStorage.Next(this.rand); + } + + public TypeKind GenerateTypeKind() + { + return (TypeKind)this.config.FieldType.Next(this.rand); + } + + public string GenerateComment() + { + int length = this.config.CommentLength.Next(this.rand); + CharDistribution distribution = this.config.UnicodeCharacters; + return this.GenerateString(length, distribution); + } + + public bool GenerateBool() + { + return this.rand.NextInt32(0, 1) != 0; + } + + public object GenerateLayoutType( + LayoutResolver resolver, + TypeArgument typeArg, + bool nullable = false, + int length = 0, + StorageKind storage = StorageKind.Sparse) + { + switch (typeArg.Type.LayoutCode) + { + case LayoutCode.Null: + return nullable && this.GenerateBool() ? (object)null : NullValue.Default; + case LayoutCode.Boolean: + return nullable && this.GenerateBool() ? (object)null : this.GenerateBool(); + case LayoutCode.Int8: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextInt8(); + case LayoutCode.Int16: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextInt16(); + case LayoutCode.Int32: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextInt32(); + case LayoutCode.Int64: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextInt64(); + case LayoutCode.UInt8: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextUInt8(); + case LayoutCode.UInt16: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextUInt16(); + case LayoutCode.UInt32: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextUInt32(); + case LayoutCode.UInt64: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextUInt64(); + case LayoutCode.VarInt: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextInt64(); + case LayoutCode.VarUInt: + return nullable && this.GenerateBool() ? (object)null : this.rand.NextUInt64(); + case LayoutCode.Float32: + return nullable && this.GenerateBool() ? (object)null : (float)this.rand.NextInt32(); + case LayoutCode.Float64: + return nullable && this.GenerateBool() ? (object)null : (double)this.rand.NextInt64(); + case LayoutCode.Float128: + return nullable && this.GenerateBool() ? (object)null : new Float128(this.rand.NextInt64(), this.rand.NextInt64()); + case LayoutCode.Decimal: + return nullable && this.GenerateBool() + ? (object)null + : new decimal( + this.rand.NextInt32(), + this.rand.NextInt32(), + this.rand.NextInt32(), + this.GenerateBool(), + this.rand.NextUInt8(0, 28)); + case LayoutCode.DateTime: + { + Contract.Assert(DateTime.MinValue.Ticks == 0); + long ticks = unchecked((long)(this.rand.NextUInt64() % (ulong)(DateTime.MaxValue.Ticks + 1))); + return nullable && this.GenerateBool() ? (object)null : new DateTime(ticks); + } + + case LayoutCode.UnixDateTime: + { + return nullable && this.GenerateBool() ? (object)null : new UnixDateTime(this.rand.NextInt64()); + } + + case LayoutCode.Guid: + return nullable && this.GenerateBool() + ? (object)null + : new Guid( + this.rand.NextInt32(), + this.rand.NextInt16(), + this.rand.NextInt16(), + this.rand.NextUInt8(), + this.rand.NextUInt8(), + this.rand.NextUInt8(), + this.rand.NextUInt8(), + this.rand.NextUInt8(), + this.rand.NextUInt8(), + this.rand.NextUInt8(), + this.rand.NextUInt8()); + + case LayoutCode.MongoDbObjectId: + { + return nullable && this.GenerateBool() ? (object)null : new MongoDbObjectId(this.rand.NextUInt32(), this.rand.NextUInt64()); + } + + case LayoutCode.Utf8: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + switch (storage) + { + case StorageKind.Variable: + length = this.rand.NextInt32(0, length == 0 ? this.config.StringValueLength.Next(this.rand) : length); + break; + case StorageKind.Sparse: + length = this.config.StringValueLength.Next(this.rand); + break; + } + + CharDistribution distribution = this.config.UnicodeCharacters; + return Utf8String.TranscodeUtf16(this.GenerateString(length, distribution)); + } + + case LayoutCode.Binary: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + switch (storage) + { + case StorageKind.Variable: + length = this.rand.NextInt32(0, length == 0 ? this.config.StringValueLength.Next(this.rand) : length); + break; + case StorageKind.Sparse: + length = this.config.StringValueLength.Next(this.rand); + break; + } + + return this.GenerateBinary(length); + } + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + Dictionary dict = new Dictionary(SamplingUtf8StringComparer.Default); + + // TODO: add properties to an object scope. + return dict; + } + + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + Contract.Assert(typeArg.TypeArgs.Count == 1); + length = this.config.CollectionValueLength.Next(this.rand); + List arrayValue = new List(length); + for (int i = 0; i < length; i++) + { + arrayValue.Add(this.GenerateLayoutType(resolver, typeArg.TypeArgs[0])); + } + + return arrayValue; + } + + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + Contract.Assert(typeArg.TypeArgs.Count == 1); + length = this.config.CollectionValueLength.Next(this.rand); + List setValue = new List(length); + for (int i = 0; i < length; i++) + { + object value = this.GenerateLayoutType(resolver, typeArg.TypeArgs[0]); + if (HybridRowValueGenerator.SetContains(resolver, setValue, value, typeArg) == -1) + { + setValue.Add(value); + } + } + + return setValue; + } + + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + Contract.Assert(typeArg.TypeArgs.Count == 2); + length = this.config.CollectionValueLength.Next(this.rand); + List> mapValue = new List>(length); + for (int i = 0; i < length; i++) + { + object key = this.GenerateLayoutType(resolver, typeArg.TypeArgs[0]); + if (HybridRowValueGenerator.MapContains(resolver, mapValue, key, typeArg) == -1) + { + object value = this.GenerateLayoutType(resolver, typeArg.TypeArgs[1]); + mapValue.Add(new List { key, value }); + } + } + + return mapValue; + } + + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + List tupleValue = new List(typeArg.TypeArgs.Count); + foreach (TypeArgument t in typeArg.TypeArgs) + { + tupleValue.Add(this.GenerateLayoutType(resolver, t)); + } + + return tupleValue; + } + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + { + if (this.GenerateBool()) + { + return null; + } + + return this.GenerateLayoutType(resolver, typeArg.TypeArgs[0]); + } + + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + { + if (nullable && this.GenerateBool()) + { + return null; + } + + SchemaId schemaId = typeArg.TypeArgs.SchemaId; + Contract.Assert(schemaId != SchemaId.Invalid); + Layout layout = resolver.Resolve(schemaId); + Contract.Assert(layout != null); + + Dictionary dict = new Dictionary(SamplingUtf8StringComparer.Default); + foreach (LayoutColumn c in layout.Columns) + { + dict[c.Path] = this.GenerateLayoutType( + resolver, + new TypeArgument(c.Type, c.TypeArgs), + length: c.Size, + storage: c.Storage); + } + + return dict; + } + + default: + Contract.Assert(false, $"Unknown type will be ignored: {typeArg}"); + return null; + } + } + + private static int SetContains(LayoutResolver resolver, List set, object right, TypeArgument typeArg) + { + for (int i = 0; i < set.Count; i++) + { + object left = set[i]; + if (HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, left, right, typeArg.TypeArgs[0])) + { + return i; + } + } + + return -1; + } + + private static int MapContains(LayoutResolver resolver, List> map, object right, TypeArgument typeArg) + { + for (int i = 0; i < map.Count; i++) + { + List pair = map[i]; + Contract.Assert(pair.Count == 2); + if (HybridRowValueGenerator.DynamicTypeArgumentEquals(resolver, pair[0], right, typeArg.TypeArgs[0])) + { + return i; + } + } + + return -1; + } + } +} diff --git a/dotnet/src/HybridRowGenerator/IntDistribution.cs b/dotnet/src/HybridRowGenerator/IntDistribution.cs new file mode 100644 index 0000000..46b964a --- /dev/null +++ b/dotnet/src/HybridRowGenerator/IntDistribution.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + + public class IntDistribution + { + private readonly int min; + private readonly int max; + private readonly DistributionType type; + + public IntDistribution(int min, int max, DistributionType type = DistributionType.Uniform) + { + this.min = min; + this.max = max; + this.type = type; + } + + public int Min => this.min; + + public int Max => this.max; + + public DistributionType Type => this.type; + + public int Next(RandomGenerator rand) + { + Contract.Requires(this.type == DistributionType.Uniform); + + return rand.NextInt32(this.min, this.max); + } + } +} diff --git a/dotnet/src/HybridRowGenerator/Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator.csproj b/dotnet/src/HybridRowGenerator/Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator.csproj new file mode 100644 index 0000000..fad2748 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator.csproj @@ -0,0 +1,23 @@ + + + + true + true + {B3F04B26-800A-4097-95CE-EACECA9ACE23} + Library + Test + Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator + Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator + netstandard2.0 + AnyCPU + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/HybridRowGenerator/Properties/AssemblyInfo.cs b/dotnet/src/HybridRowGenerator/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..b5a6fb8 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("B3F04B26-800A-4097-95CE-EACECA9ACE23")] diff --git a/dotnet/src/HybridRowGenerator/RandomGenerator.cs b/dotnet/src/HybridRowGenerator/RandomGenerator.cs new file mode 100644 index 0000000..3ffb71d --- /dev/null +++ b/dotnet/src/HybridRowGenerator/RandomGenerator.cs @@ -0,0 +1,131 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using System.Runtime.InteropServices; + using Microsoft.Azure.Cosmos.Core; + + public class RandomGenerator + { + private readonly Random root; + + public RandomGenerator(Random root) + { + this.root = root; + } + + /// Returns a uniformly distributed 8-bit signed integer in the range specified. + /// The inclusive lower bound of the random number returned. + /// The inclusive upper bound of the random number returned. + /// Requires < . + public sbyte NextInt8(sbyte min = sbyte.MinValue, sbyte max = sbyte.MaxValue) + { + Contract.Requires(min <= max); + + unchecked + { + sbyte result = (sbyte)(this.NextUInt8(0, (byte)(max - min)) + min); + return result; + } + } + + /// Returns a uniformly distributed 16-bit signed integer in the range specified. + /// The inclusive lower bound of the random number returned. + /// The inclusive upper bound of the random number returned. + /// Requires < . + public short NextInt16(short min = short.MinValue, short max = short.MaxValue) + { + Contract.Requires(min <= max); + + unchecked + { + short result = (short)(this.NextUInt16(0, (ushort)(max - min)) + min); + return result; + } + } + + /// Returns a uniformly distributed 32-bit signed integer in the range specified. + /// The inclusive lower bound of the random number returned. + /// The inclusive upper bound of the random number returned. + /// Requires < . + public int NextInt32(int min = int.MinValue, int max = int.MaxValue) + { + Contract.Requires(min <= max); + + unchecked + { + int result = (int)(this.NextUInt32(0, (uint)(max - min)) + min); + return result; + } + } + + /// Returns a uniformly distributed 64-bit signed integer. + public long NextInt64() + { + unchecked + { + long result = (long)this.NextUInt64(); + return result; + } + } + + /// Returns a uniformly distributed 8-bit unsigned integer in the range specified. + /// The inclusive lower bound of the random number returned. + /// The inclusive upper bound of the random number returned. + /// Requires < . + public byte NextUInt8(byte min = byte.MinValue, byte max = byte.MaxValue) + { + Contract.Requires(min <= max); + + ulong result = this.NextUInt64(); + unchecked + { + result = (result % (((ulong)max - (ulong)min) + 1)) + (ulong)min; + return (byte)result; + } + } + + /// Returns a uniformly distributed 16-bit unsigned integer in the range specified. + /// The inclusive lower bound of the random number returned. + /// The inclusive upper bound of the random number returned. + /// Requires < . + public ushort NextUInt16(ushort min = ushort.MinValue, ushort max = ushort.MaxValue) + { + Contract.Requires(min <= max); + + ulong result = this.NextUInt64(); + unchecked + { + result = (result % (((ulong)max - (ulong)min) + 1)) + (ulong)min; + return (ushort)result; + } + } + + /// Returns a uniformly distributed 32-bit unsigned integer in the range specified. + /// The inclusive lower bound of the random number returned. + /// The inclusive upper bound of the random number returned. + /// Requires < . + public uint NextUInt32(uint min = uint.MinValue, uint max = uint.MaxValue) + { + Contract.Requires(min <= max); + + ulong result = this.NextUInt64(); + unchecked + { + result = (result % (((ulong)max - (ulong)min) + 1)) + (ulong)min; + return (uint)result; + } + } + + /// Returns a uniformly distributed 64-bit unsigned integer. + public ulong NextUInt64() + { + Span result = stackalloc ulong[1]; + this.root.NextBytes(MemoryMarshal.Cast(result)); + return result[0]; + } + } +} diff --git a/dotnet/src/HybridRowGenerator/SchemaGenerator.cs b/dotnet/src/HybridRowGenerator/SchemaGenerator.cs new file mode 100644 index 0000000..494a493 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/SchemaGenerator.cs @@ -0,0 +1,277 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public class SchemaGenerator + { + private readonly RandomGenerator rand; + private readonly HybridRowGeneratorConfig config; + private readonly HybridRowValueGenerator generator; + + public SchemaGenerator(RandomGenerator rand, HybridRowGeneratorConfig config, HybridRowValueGenerator generator) + { + this.rand = rand; + this.config = config; + this.generator = generator; + } + + public Namespace InitializeRandomNamespace() + { + Namespace ns = new Namespace() + { + Name = this.generator.GenerateIdentifier(), + }; + + return ns; + } + + public Schema InitializeRandomSchema(Namespace ns, int depth) + { + string name = HybridRowValueGenerator.GenerateExclusive( + this.generator.GenerateIdentifier, + from s1 in ns.Schemas select s1.Name, + this.config.ConflictRetryAttempts); + + SchemaId sid = HybridRowValueGenerator.GenerateExclusive( + this.generator.GenerateSchemaId, + from s1 in ns.Schemas select s1.SchemaId, + this.config.ConflictRetryAttempts); + + // Allocate and insert the schema *before* recursing to allocate properties. This ensures that nested structure + // doesn't conflict with name or id constraints. + Schema s = new Schema() + { + Name = name, + SchemaId = sid, + Type = TypeKind.Schema, + Comment = this.generator.GenerateComment(), + Options = this.InitializeRandomSchemaOptions(), + }; + + ns.Schemas.Add(s); + + // Recurse and allocate its properties. + s.Properties = this.InitializeRandomProperties(ns, TypeKind.Schema, depth); + + return s; + } + + private List InitializeRandomProperties(Namespace ns, TypeKind scope, int depth) + { + int length = this.config.NumTableProperties.Next(this.rand); + + // Introduce some decay rate for the number of properties in nested contexts + // to limit the depth of schemas. + if (depth > 0) + { + double scaled = Math.Floor(length * Math.Exp(depth * this.config.DepthDecayFactor)); + Contract.Assert(scaled <= length); + length = (int)scaled; + } + + List properties = new List(length); + for (int i = 0; i < length; i++) + { + PropertyType propType = this.InitializeRandomPropertyType(ns, scope, depth); + string path = HybridRowValueGenerator.GenerateExclusive( + this.generator.GenerateIdentifier, + from s1 in properties select s1.Path, + this.config.ConflictRetryAttempts); + Property prop = new Property() + { + Comment = this.generator.GenerateComment(), + Path = path, + PropertyType = propType, + }; + + properties.Add(prop); + } + + return properties; + } + + private PropertyType InitializeRandomPropertyType(Namespace ns, TypeKind scope, int depth) + { + TypeKind type = this.generator.GenerateTypeKind(); + PropertyType propType; + switch (type) + { + case TypeKind.Object: + propType = new ObjectPropertyType() + { + Immutable = this.generator.GenerateBool(), + + // TODO: add properties to object scopes. + // Properties = this.InitializeRandomProperties(ns, type, depth + 1), + }; + + break; + case TypeKind.Array: + propType = new ArrayPropertyType() + { + Immutable = this.generator.GenerateBool(), + Items = this.InitializeRandomPropertyType(ns, type, depth + 1), + }; + + break; + case TypeKind.Set: + propType = new SetPropertyType() + { + Immutable = this.generator.GenerateBool(), + Items = this.InitializeRandomPropertyType(ns, type, depth + 1), + }; + + break; + case TypeKind.Map: + propType = new MapPropertyType() + { + Immutable = this.generator.GenerateBool(), + Keys = this.InitializeRandomPropertyType(ns, type, depth + 1), + Values = this.InitializeRandomPropertyType(ns, type, depth + 1), + }; + + break; + case TypeKind.Tuple: + int numItems = this.config.NumTupleItems.Next(this.rand); + List itemTypes = new List(numItems); + for (int i = 0; i < numItems; i++) + { + itemTypes.Add(this.InitializeRandomPropertyType(ns, type, depth + 1)); + } + + propType = new TuplePropertyType() + { + Immutable = this.generator.GenerateBool(), + Items = itemTypes, + }; + + break; + case TypeKind.Tagged: + int numTagged = this.config.NumTaggedItems.Next(this.rand); + List taggedItemTypes = new List(numTagged); + for (int i = 0; i < numTagged; i++) + { + taggedItemTypes.Add(this.InitializeRandomPropertyType(ns, type, depth + 1)); + } + + propType = new TaggedPropertyType() + { + Immutable = this.generator.GenerateBool(), + Items = taggedItemTypes, + }; + + break; + case TypeKind.Schema: + Schema udt = this.InitializeRandomSchema(ns, depth + 1); + propType = new UdtPropertyType() + { + Immutable = this.generator.GenerateBool(), + Name = udt.Name, + }; + + break; + default: + StorageKind storage = (scope == TypeKind.Schema) ? this.generator.GenerateStorageKind() : StorageKind.Sparse; + switch (storage) + { + case StorageKind.Sparse: + // All types are supported in Sparse. + break; + + case StorageKind.Fixed: + switch (type) + { + case TypeKind.Null: + case TypeKind.Boolean: + case TypeKind.Int8: + case TypeKind.Int16: + case TypeKind.Int32: + case TypeKind.Int64: + case TypeKind.UInt8: + case TypeKind.UInt16: + case TypeKind.UInt32: + case TypeKind.UInt64: + case TypeKind.Float32: + case TypeKind.Float64: + case TypeKind.Float128: + case TypeKind.Decimal: + case TypeKind.DateTime: + case TypeKind.UnixDateTime: + case TypeKind.Guid: + case TypeKind.MongoDbObjectId: + case TypeKind.Utf8: + case TypeKind.Binary: + // Only these types are supported with fixed storage today. + break; + default: + storage = StorageKind.Sparse; + break; + } + + break; + case StorageKind.Variable: + switch (type) + { + case TypeKind.Binary: + case TypeKind.Utf8: + case TypeKind.VarInt: + case TypeKind.VarUInt: + // Only these types are supported with variable storage today. + break; + + default: + storage = StorageKind.Sparse; + break; + } + + break; + } + + propType = new PrimitivePropertyType() + { + Length = storage == StorageKind.Sparse ? 0 : this.config.PrimitiveFieldValueLength.Next(this.rand), + Storage = storage, + }; + + break; + } + + propType.ApiType = this.generator.GenerateIdentifier(); + propType.Type = type; + switch (scope) + { + case TypeKind.Array: + case TypeKind.Map: + case TypeKind.Set: + case TypeKind.Tuple: + case TypeKind.Tagged: + propType.Nullable = this.generator.GenerateBool(); + break; + default: + propType.Nullable = true; + break; + } + + return propType; + } + + private SchemaOptions InitializeRandomSchemaOptions() + { + SchemaOptions o = new SchemaOptions() + { + DisallowUnschematized = this.generator.GenerateBool(), + }; + + return o; + } + } +} diff --git a/dotnet/src/HybridRowGenerator/StreamingRowGenerator.cs b/dotnet/src/HybridRowGenerator/StreamingRowGenerator.cs new file mode 100644 index 0000000..553099b --- /dev/null +++ b/dotnet/src/HybridRowGenerator/StreamingRowGenerator.cs @@ -0,0 +1,330 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + + public ref struct StreamingRowGenerator + { + private RowBuffer row; + + public StreamingRowGenerator(int capacity, Layout layout, LayoutResolver resolver, ISpanResizer resizer = default) + { + this.row = new RowBuffer(capacity, resizer); + this.row.InitLayout(HybridRowVersion.V1, layout, resolver); + } + + public int Length => this.row.Length; + + public byte[] ToArray() => this.row.ToArray(); + + public void WriteTo(Stream stream) + { + this.row.WriteTo(stream); + } + + public bool ReadFrom(Stream stream, int length) + { + return this.row.ReadFrom(stream, length, HybridRowVersion.V1, this.row.Resolver); + } + + public void Reset() + { + Layout layout = this.row.Resolver.Resolve(this.row.Header.SchemaId); + this.row.InitLayout(HybridRowVersion.V1, layout, this.row.Resolver); + } + + public RowReader GetReader() + { + return new RowReader(ref this.row); + } + + public Result WriteBuffer(Dictionary value) + { + return RowWriter.WriteBuffer( + ref this.row, + value, + (ref RowWriter writer, TypeArgument typeArg, Dictionary dict) => + { + Layout layout = writer.Resolver.Resolve(typeArg.TypeArgs.SchemaId); + foreach (LayoutColumn c in layout.Columns) + { + Result result = StreamingRowGenerator.LayoutCodeSwitch(ref writer, c.Path, c.TypeArg, dict[c.Path]); + if (result != Result.Success) + { + return result; + } + } + + return Result.Success; + }); + } + + private static Result LayoutCodeSwitch(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + switch (typeArg.Type.LayoutCode) + { + case LayoutCode.Null: + return writer.WriteNull(path); + + case LayoutCode.Boolean: + return writer.WriteBool(path, value == null ? default : (bool)value); + + case LayoutCode.Int8: + return writer.WriteInt8(path, value == null ? default : (sbyte)value); + + case LayoutCode.Int16: + return writer.WriteInt16(path, value == null ? default : (short)value); + + case LayoutCode.Int32: + return writer.WriteInt32(path, value == null ? default : (int)value); + + case LayoutCode.Int64: + return writer.WriteInt64(path, value == null ? default : (long)value); + + case LayoutCode.UInt8: + return writer.WriteUInt8(path, value == null ? default : (byte)value); + + case LayoutCode.UInt16: + return writer.WriteUInt16(path, value == null ? default : (ushort)value); + + case LayoutCode.UInt32: + return writer.WriteUInt32(path, value == null ? default : (uint)value); + + case LayoutCode.UInt64: + return writer.WriteUInt64(path, value == null ? default : (ulong)value); + + case LayoutCode.VarInt: + return writer.WriteVarInt(path, value == null ? default : (long)value); + + case LayoutCode.VarUInt: + return writer.WriteVarUInt(path, value == null ? default : (ulong)value); + + case LayoutCode.Float32: + return writer.WriteFloat32(path, value == null ? default : (float)value); + + case LayoutCode.Float64: + return writer.WriteFloat64(path, value == null ? default : (double)value); + + case LayoutCode.Float128: + return writer.WriteFloat128(path, value == null ? default : (Float128)value); + + case LayoutCode.Decimal: + return writer.WriteDecimal(path, value == null ? default : (decimal)value); + + case LayoutCode.DateTime: + return writer.WriteDateTime(path, value == null ? default : (DateTime)value); + + case LayoutCode.UnixDateTime: + return writer.WriteUnixDateTime(path, value == null ? default : (UnixDateTime)value); + + case LayoutCode.Guid: + return writer.WriteGuid(path, value == null ? default : (Guid)value); + + case LayoutCode.MongoDbObjectId: + return writer.WriteMongoDbObjectId(path, value == null ? default : (MongoDbObjectId)value); + + case LayoutCode.Utf8: + return writer.WriteString(path, value == null ? default : (Utf8String)value); + + case LayoutCode.Binary: + return writer.WriteBinary(path, value == null ? default : (byte[])value); + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + return StreamingRowGenerator.DispatchObject(ref writer, path, typeArg, value); + + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + return StreamingRowGenerator.DispatchArray(ref writer, path, typeArg, value); + + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + return StreamingRowGenerator.DispatchSet(ref writer, path, typeArg, value); + + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + return StreamingRowGenerator.DispatchMap(ref writer, path, typeArg, value); + + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + return StreamingRowGenerator.DispatchTuple(ref writer, path, typeArg, value); + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + return StreamingRowGenerator.DispatchNullable(ref writer, path, typeArg, value); + + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + return StreamingRowGenerator.DispatchUDT(ref writer, path, typeArg, value); + + default: + Contract.Assert(false, $"Unknown type will be ignored: {typeArg}"); + return Result.Failure; + } + } + + private static Result DispatchObject(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + return writer.WriteScope( + path, + typeArg, + (Dictionary)value, + (ref RowWriter writer2, TypeArgument typeArg2, Dictionary value2) => + { + // TODO: support properties in an object scope. + return Result.Success; + }); + } + + private static Result DispatchArray(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 1); + + return writer.WriteScope( + path, + typeArg, + (List)value, + (ref RowWriter writer2, TypeArgument typeArg2, List items2) => + { + foreach (object item in items2) + { + Result r = StreamingRowGenerator.LayoutCodeSwitch(ref writer2, null, typeArg2.TypeArgs[0], item); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + }); + } + + private static Result DispatchTuple(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count >= 2); + List items = (List)value; + Contract.Assert(items.Count == typeArg.TypeArgs.Count); + + return writer.WriteScope( + path, + typeArg, + items, + (ref RowWriter writer2, TypeArgument typeArg2, List items2) => + { + for (int i = 0; i < items2.Count; i++) + { + object item = items2[i]; + Result r = StreamingRowGenerator.LayoutCodeSwitch(ref writer2, null, typeArg2.TypeArgs[i], item); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + }); + } + + private static Result DispatchNullable(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 1); + + RowWriter.WriterFunc f0 = null; + if (value != null) + { + f0 = (ref RowWriter writer2, TypeArgument typeArg2, object value2) => + StreamingRowGenerator.LayoutCodeSwitch(ref writer2, null, typeArg2.TypeArgs[0], value2); + } + + return writer.WriteScope(path, typeArg, value, f0); + } + + private static Result DispatchSet(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 1); + + return writer.WriteScope( + path, + typeArg, + (List)value, + (ref RowWriter writer2, TypeArgument typeArg2, List items2) => + { + foreach (object item in items2) + { + Result r = StreamingRowGenerator.LayoutCodeSwitch(ref writer2, null, typeArg2.TypeArgs[0], item); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + }); + } + + private static Result DispatchMap(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + Contract.Requires(typeArg.TypeArgs.Count == 2); + + return writer.WriteScope( + path, + typeArg, + (List)value, + (ref RowWriter writer2, TypeArgument typeArg2, List items2) => + { + TypeArgument fieldType = new TypeArgument( + typeArg2.Type.Immutable ? LayoutType.ImmutableTypedTuple : LayoutType.TypedTuple, + typeArg2.TypeArgs); + + foreach (object item in items2) + { + Result r = StreamingRowGenerator.DispatchTuple(ref writer2, null, fieldType, item); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + }); + } + + private static Result DispatchUDT(ref RowWriter writer, Utf8String path, TypeArgument typeArg, object value) + { + return writer.WriteScope( + path, + typeArg, + (Dictionary)value, + (ref RowWriter writer2, TypeArgument typeArg2, Dictionary dict) => + { + Layout udt = writer2.Resolver.Resolve(typeArg2.TypeArgs.SchemaId); + foreach (LayoutColumn c in udt.Columns) + { + Result result = StreamingRowGenerator.LayoutCodeSwitch(ref writer2, c.Path, c.TypeArg, dict[c.Path]); + if (result != Result.Success) + { + return result; + } + } + + return Result.Success; + }); + } + } +} diff --git a/dotnet/src/HybridRowGenerator/VisitRowGenerator.cs b/dotnet/src/HybridRowGenerator/VisitRowGenerator.cs new file mode 100644 index 0000000..14c1372 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/VisitRowGenerator.cs @@ -0,0 +1,260 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + public ref struct VisitRowGenerator + { + private readonly LayoutResolver resolver; + private RowBuffer row; + + public VisitRowGenerator(Span span, LayoutResolver resolver) + { + this.resolver = resolver; + this.row = new RowBuffer(span, HybridRowVersion.V1, this.resolver); + } + + public int Length => this.row.Length; + + public Result DispatchLayout(Layout layout) + { + RowCursor scope = RowCursor.Create(ref this.row); + return this.DispatchLayout(ref scope, layout); + } + + private Result LayoutCodeSwitch( + ref RowCursor scope, + LayoutColumn col = default, + TypeArgument typeArg = default) + { + if (col != null) + { + typeArg = col.TypeArg; + } + + switch (typeArg.Type.LayoutCode) + { + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + return this.DispatchObject(ref scope, typeArg); + + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + return this.DispatchArray(ref scope, typeArg); + + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + return this.DispatchSet(ref scope, typeArg); + + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + return this.DispatchMap(ref scope, typeArg); + + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + return this.DispatchTuple(ref scope, typeArg); + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + return this.DispatchNullable(ref scope, typeArg); + + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + return this.DispatchUDT(ref scope, typeArg); + + default: + return Result.Success; + } + } + + private Result DispatchObject(ref RowCursor scope, TypeArgument t) + { + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + // TODO: support properties in an object scope. + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchArray(ref RowCursor scope, TypeArgument t) + { + Contract.Assert(t.TypeArgs.Count == 1); + + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + while (childScope.MoveNext(ref this.row)) + { + r = this.LayoutCodeSwitch(ref childScope, null, t.TypeArgs[0]); + if (r != Result.Success) + { + return r; + } + } + + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchTuple(ref RowCursor scope, TypeArgument t) + { + Contract.Assert(t.TypeArgs.Count >= 2); + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + while (childScope.MoveNext(ref this.row)) + { + r = this.LayoutCodeSwitch(ref childScope, null, t.TypeArgs[childScope.Index]); + if (r != Result.Success) + { + return r; + } + } + + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchNullable(ref RowCursor scope, TypeArgument t) + { + Contract.Assert(t.TypeArgs.Count == 1); + + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + if (childScope.MoveNext(ref this.row)) + { + r = this.LayoutCodeSwitch(ref childScope, null, t.TypeArgs[0]); + if (r != Result.Success) + { + return r; + } + } + + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchSet(ref RowCursor scope, TypeArgument t) + { + Contract.Assert(t.TypeArgs.Count == 1); + + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + while (childScope.MoveNext(ref this.row)) + { + r = this.LayoutCodeSwitch(ref childScope, null, t.TypeArgs[0]); + if (r != Result.Success) + { + return r; + } + } + + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchMap(ref RowCursor scope, TypeArgument t) + { + Contract.Assert(t.TypeArgs.Count == 2); + + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + TypeArgument fieldType = t.TypeAs().FieldType(ref childScope); + while (childScope.MoveNext(ref this.row)) + { + r = this.LayoutCodeSwitch(ref childScope, null, fieldType); + if (r != Result.Success) + { + return r; + } + } + + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchUDT(ref RowCursor scope, TypeArgument t) + { + Result r = t.TypeAs().ReadScope(ref this.row, ref scope, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + Layout layout = this.resolver.Resolve(t.TypeArgs.SchemaId); + r = this.DispatchLayout(ref childScope, layout); + if (r != Result.Success) + { + return r; + } + + scope.Skip(ref this.row, ref childScope); + return Result.Success; + } + + private Result DispatchLayout(ref RowCursor scope, Layout layout) + { + // Process schematized segment. + foreach (LayoutColumn c in layout.Columns) + { + if (c.Storage == StorageKind.Sparse) + { + break; + } + + Result r = this.LayoutCodeSwitch(ref scope, c); + if (r != Result.Success) + { + return r; + } + } + + // Process sparse segment. + while (scope.MoveNext(ref this.row)) + { + Result r = this.LayoutCodeSwitch(ref scope, null, scope.TypeArg); + if (r != Result.Success) + { + return r; + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRowGenerator/WriteRowGenerator.cs b/dotnet/src/HybridRowGenerator/WriteRowGenerator.cs new file mode 100644 index 0000000..92a9fc1 --- /dev/null +++ b/dotnet/src/HybridRowGenerator/WriteRowGenerator.cs @@ -0,0 +1,502 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + + [SuppressMessage("Microsoft.StyleCop.CSharp.OrderingRules", "SA1401", Justification = "Test types.")] + public ref struct WriteRowGenerator + { + private RowBuffer row; + + public WriteRowGenerator(int capacity, Layout layout, LayoutResolver resolver) + { + this.row = new RowBuffer(capacity); + this.row.InitLayout(HybridRowVersion.V1, layout, resolver); + } + + public int Length => this.row.Length; + + public void Reset() + { + Layout layout = this.row.Resolver.Resolve(this.row.Header.SchemaId); + this.row.InitLayout(HybridRowVersion.V1, layout, this.row.Resolver); + } + + public RowReader GetReader() + { + return new RowReader(ref this.row); + } + + public Result DispatchLayout(Layout layout, Dictionary dict) + { + RowCursor scope = RowCursor.Create(ref this.row); + return WriteRowGenerator.DispatchLayout(ref this.row, ref scope, layout, dict); + } + + private static Result LayoutCodeSwitch( + ref RowBuffer row, + ref RowCursor scope, + LayoutColumn col = default, + TypeArgument typeArg = default, + object value = null) + { + if (col != null) + { + typeArg = col.TypeArg; + } + + // ReSharper disable MergeConditionalExpression + // ReSharper disable SimplifyConditionalTernaryExpression + // ReSharper disable RedundantTypeSpecificationInDefaultExpression +#pragma warning disable IDE0034 // Simplify 'default' expression + switch (typeArg.Type.LayoutCode) + { + case LayoutCode.Null: + return WriteRowGenerator.Dispatch(ref row, ref scope, col, typeArg.Type, NullValue.Default); + + case LayoutCode.Boolean: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(bool) : (bool)value); + + case LayoutCode.Int8: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(sbyte) : (sbyte)value); + + case LayoutCode.Int16: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(short) : (short)value); + + case LayoutCode.Int32: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(int) : (int)value); + + case LayoutCode.Int64: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(long) : (long)value); + + case LayoutCode.UInt8: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(byte) : (byte)value); + + case LayoutCode.UInt16: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(ushort) : (ushort)value); + + case LayoutCode.UInt32: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(uint) : (uint)value); + + case LayoutCode.UInt64: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(ulong) : (ulong)value); + + case LayoutCode.VarInt: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(long) : (long)value); + + case LayoutCode.VarUInt: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(ulong) : (ulong)value); + + case LayoutCode.Float32: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(float) : (float)value); + + case LayoutCode.Float64: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(double) : (double)value); + + case LayoutCode.Float128: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(Float128) : (Float128)value); + + case LayoutCode.Decimal: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(decimal) : (decimal)value); + + case LayoutCode.DateTime: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(DateTime) : (DateTime)value); + + case LayoutCode.UnixDateTime: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(UnixDateTime) : (UnixDateTime)value); + + case LayoutCode.Guid: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(Guid) : (Guid)value); + + case LayoutCode.MongoDbObjectId: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(MongoDbObjectId) : (MongoDbObjectId)value); + + case LayoutCode.Utf8: + return WriteRowGenerator.Dispatch(ref row, ref scope, col, (Utf8String)value); + + case LayoutCode.Binary: + return WriteRowGenerator.Dispatch( + ref row, + ref scope, + col, + typeArg.Type, + value == null ? default(byte[]) : (byte[])value); + + case LayoutCode.ObjectScope: + case LayoutCode.ImmutableObjectScope: + return WriteRowGenerator.DispatchObject(ref row, ref scope, typeArg, value); + + case LayoutCode.TypedArrayScope: + case LayoutCode.ImmutableTypedArrayScope: + return WriteRowGenerator.DispatchArray(ref row, ref scope, typeArg, value); + + case LayoutCode.TypedSetScope: + case LayoutCode.ImmutableTypedSetScope: + return WriteRowGenerator.DispatchSet(ref row, ref scope, typeArg, value); + + case LayoutCode.TypedMapScope: + case LayoutCode.ImmutableTypedMapScope: + return WriteRowGenerator.DispatchMap(ref row, ref scope, typeArg, value); + + case LayoutCode.TupleScope: + case LayoutCode.ImmutableTupleScope: + case LayoutCode.TypedTupleScope: + case LayoutCode.ImmutableTypedTupleScope: + case LayoutCode.TaggedScope: + case LayoutCode.ImmutableTaggedScope: + case LayoutCode.Tagged2Scope: + case LayoutCode.ImmutableTagged2Scope: + return WriteRowGenerator.DispatchTuple(ref row, ref scope, typeArg, value); + + case LayoutCode.NullableScope: + case LayoutCode.ImmutableNullableScope: + return WriteRowGenerator.DispatchNullable(ref row, ref scope, typeArg, value); + + case LayoutCode.Schema: + case LayoutCode.ImmutableSchema: + return WriteRowGenerator.DispatchUDT(ref row, ref scope, typeArg, value); + + default: + Contract.Assert(false, $"Unknown type will be ignored: {typeArg.Type.LayoutCode}"); + return Result.Failure; + } + + // ReSharper restore SimplifyConditionalTernaryExpression + // ReSharper restore MergeConditionalExpression + // ReSharper restore RedundantTypeSpecificationInDefaultExpression +#pragma warning restore IDE0034 // Simplify 'default' expression + } + + private static Result Dispatch(ref RowBuffer row, ref RowCursor root, LayoutColumn col, LayoutType t, TValue value) + where TLayout : LayoutType + { + switch (col?.Storage) + { + case StorageKind.Fixed: + return t.TypeAs().WriteFixed(ref row, ref root, col, value); + + case StorageKind.Variable: + return t.TypeAs().WriteVariable(ref row, ref root, col, value); + + default: + return t.TypeAs().WriteSparse(ref row, ref root, value); + } + } + + private static Result Dispatch(ref RowBuffer row, ref RowCursor root, LayoutColumn col, Utf8String value) + { + switch (col?.Storage) + { + case StorageKind.Fixed: + return LayoutType.Utf8.WriteFixed(ref row, ref root, col, value); + + case StorageKind.Variable: + return LayoutType.Utf8.WriteVariable(ref row, ref root, col, value); + + default: + return LayoutType.Utf8.WriteSparse(ref row, ref root, value); + } + } + + private static Result DispatchObject(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Result r = t.TypeAs().WriteScope(ref row, ref scope, t.TypeArgs, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + // TODO: support properties in an object scope. + Dictionary dict = (Dictionary)value; + Contract.Assert(dict.Count == 0); + scope.Skip(ref row, ref childScope); + return Result.Success; + } + + private static Result DispatchArray(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Contract.Assert(t.TypeArgs.Count == 1); + + Result r = t.TypeAs().WriteScope(ref row, ref scope, t.TypeArgs, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + List items = (List)value; + foreach (object item in items) + { + r = WriteRowGenerator.LayoutCodeSwitch(ref row, ref childScope, null, t.TypeArgs[0], item); + if (r != Result.Success) + { + return r; + } + + childScope.MoveNext(ref row); + } + + scope.Skip(ref row, ref childScope); + return Result.Success; + } + + private static Result DispatchTuple(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Contract.Assert(t.TypeArgs.Count >= 2); + + return t.TypeAs() + .WriteScope( + ref row, + ref scope, + t.TypeArgs, + (List)value, + (ref RowBuffer row2, ref RowCursor childScope, List items) => + { + Contract.Assert(items.Count == childScope.ScopeTypeArgs.Count); + for (int i = 0; i < items.Count; i++) + { + Result r = WriteRowGenerator.LayoutCodeSwitch(ref row2, ref childScope, null, childScope.ScopeTypeArgs[i], items[i]); + if (r != Result.Success) + { + return r; + } + + childScope.MoveNext(ref row2); + } + + return Result.Success; + }); + } + + private static Result DispatchNullable(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Contract.Assert(t.TypeArgs.Count == 1); + + bool hasValue = value != null; + Result r = t.TypeAs().WriteScope(ref row, ref scope, t.TypeArgs, hasValue, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + if (hasValue) + { + r = WriteRowGenerator.LayoutCodeSwitch(ref row, ref childScope, null, t.TypeArgs[0], value); + if (r != Result.Success) + { + return r; + } + + childScope.MoveNext(ref row); + } + + scope.Skip(ref row, ref childScope); + return Result.Success; + } + + private static Result DispatchSet(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Contract.Assert(t.TypeArgs.Count == 1); + + return t.TypeAs() + .WriteScope( + ref row, + ref scope, + t.TypeArgs, + (List)value, + (ref RowBuffer row2, ref RowCursor childScope, List items) => + { + foreach (object item in items) + { + Result r = WriteRowGenerator.LayoutCodeSwitch(ref row2, ref childScope, null, childScope.ScopeTypeArgs[0], item); + if (r != Result.Success) + { + return r; + } + + childScope.MoveNext(ref row2); + } + + return Result.Success; + }); + } + + private static Result DispatchMap(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Contract.Assert(t.TypeArgs.Count == 2); + return t.TypeAs() + .WriteScope( + ref row, + ref scope, + t.TypeArgs, + (List)value, + (ref RowBuffer row2, ref RowCursor childScope, List pairs) => + { + TypeArgument fieldType = childScope.ScopeType.TypeAs().FieldType(ref childScope); + foreach (object elm in pairs) + { + Result r = WriteRowGenerator.LayoutCodeSwitch(ref row2, ref childScope, null, fieldType, elm); + if (r != Result.Success) + { + return r; + } + + childScope.MoveNext(ref row2); + } + + return Result.Success; + }); + } + + private static Result DispatchUDT(ref RowBuffer row, ref RowCursor scope, TypeArgument t, object value) + { + Result r = t.TypeAs().WriteScope(ref row, ref scope, t.TypeArgs, out RowCursor childScope); + if (r != Result.Success) + { + return r; + } + + Dictionary dict = (Dictionary)value; + Layout layout = row.Resolver.Resolve(t.TypeArgs.SchemaId); + r = WriteRowGenerator.DispatchLayout(ref row, ref childScope, layout, dict); + if (r != Result.Success) + { + return r; + } + + scope.Skip(ref row, ref childScope); + return Result.Success; + } + + private static Result DispatchLayout(ref RowBuffer row, ref RowCursor scope, Layout layout, Dictionary dict) + { + foreach (LayoutColumn c in layout.Columns) + { + if (c.Storage != StorageKind.Sparse) + { + Result r = WriteRowGenerator.LayoutCodeSwitch(ref row, ref scope, c, value: dict[c.Path]); + if (r != Result.Success) + { + return r; + } + } + else + { + scope.Find(ref row, c.Path); + Result r = WriteRowGenerator.LayoutCodeSwitch(ref row, ref scope, null, c.TypeArg, dict[c.Path]); + if (r != Result.Success) + { + return r; + } + } + } + + return Result.Success; + } + } +} diff --git a/dotnet/src/HybridRowStress/HybridRowStressConfig.cs b/dotnet/src/HybridRowStress/HybridRowStressConfig.cs new file mode 100644 index 0000000..9a42b84 --- /dev/null +++ b/dotnet/src/HybridRowStress/HybridRowStressConfig.cs @@ -0,0 +1,157 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowStress +{ + using System; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.Extensions.CommandLineUtils; + + public class HybridRowStressConfig + { + private HybridRowStressConfig() + { + } + + public HybridRowGeneratorConfig GeneratorConfig { get; set; } + + public bool Verbose { get; set; } = false; + + public bool BreakOnError { get; set; } = false; + + public long Iterations { get; set; } = -1; + + public long InnerLoopCount { get; set; } = 1; + + public bool GCBeforeInnerLoop { get; set; } = false; + + public int Seed { get; set; } = 42; + + public long ReportFrequency { get; set; } = 1000; + + public string NamespaceFile { get; set; } = null; + + public string TableSchemaName { get; set; } = null; + + public string ValueFile { get; set; } = null; + + public static HybridRowStressConfig FromArgs(string[] args) + { + CommandLineApplication command = new CommandLineApplication(throwOnUnexpectedArg: true) + { + Name = nameof(HybridRowStressProgram), + Description = "HybridRow Stress.", + }; + + HybridRowStressConfig config = new HybridRowStressConfig + { + GeneratorConfig = new HybridRowGeneratorConfig(), + }; + + command.HelpOption("-? | -h | --help"); + CommandOption verboseOpt = command.Option("-verbose", "Display verbose output. Default: false.", CommandOptionType.NoValue); + CommandOption breakOnErrOpt = command.Option("-breakonerror", "Stop after the first error. Default: false.", CommandOptionType.NoValue); + CommandOption namespaceOpt = command.Option( + "-namespace", + "Force stress to used the fixed schema Namespace. Default: null.", + CommandOptionType.SingleValue); + + CommandOption tableNameOpt = command.Option( + "-tablename", + "The table schema (when using -namespace). Default: null.", + CommandOptionType.SingleValue); + + CommandOption valueOpt = command.Option( + "-value", + "Force stress to use the value encoded in the file as JSON. Default: null.", + CommandOptionType.SingleValue); + + CommandOption seedOpt = command.Option( + "-seed", + " Random number generator seed. Default: 42.", + CommandOptionType.SingleValue); + + CommandOption randomOpt = command.Option("-random", "Choose a random seed for the number generator. Default: false.", CommandOptionType.NoValue); + + CommandOption iterationsOpt = command.Option( + "-iterations", + " Number of test iterations. Default: unlimited.", + CommandOptionType.SingleValue); + + CommandOption innerloopOpt = command.Option( + "-innerloop", + " Number of times to write a row in an inner. Default: 1.", + CommandOptionType.SingleValue); + + CommandOption gcOpt = command.Option("-gc", "Perform a forced GC before executing inner loop encoding.", CommandOptionType.NoValue); + + CommandOption reportFrequencyOpt = command.Option( + "-reportfrequency", + " Report progress every N iterations. Default: 1000.", + CommandOptionType.SingleValue); + + command.OnExecute( + () => + { + config.Verbose = verboseOpt.HasValue(); + config.BreakOnError = breakOnErrOpt.HasValue(); + if (namespaceOpt.HasValue()) + { + config.NamespaceFile = namespaceOpt.Value(); + } + + if (tableNameOpt.HasValue()) + { + config.TableSchemaName = tableNameOpt.Value(); + } + + if (valueOpt.HasValue()) + { + config.ValueFile = valueOpt.Value(); + } + + if (seedOpt.HasValue()) + { + config.Seed = int.Parse(seedOpt.Value()); + } + + if (randomOpt.HasValue()) + { + config.Seed = unchecked((int)DateTime.UtcNow.Ticks); + } + + if (iterationsOpt.HasValue()) + { + config.Iterations = int.Parse(iterationsOpt.Value()); + } + + if (innerloopOpt.HasValue()) + { + config.InnerLoopCount = int.Parse(innerloopOpt.Value()); + } + + config.GCBeforeInnerLoop = gcOpt.HasValue(); + + if (reportFrequencyOpt.HasValue()) + { + config.ReportFrequency = long.Parse(reportFrequencyOpt.Value()); + if (config.ReportFrequency <= 0) + { + throw new CommandParsingException(command, "Report Frequency cannot be negative."); + } + } + + return 0; + }); + + command.Execute(args); + if (command.IsShowingInformation) + { + throw new CommandParsingException(command, "Help"); + } + + return config; + } + } +} diff --git a/dotnet/src/HybridRowStress/HybridRowStressProgram.cs b/dotnet/src/HybridRowStress/HybridRowStressProgram.cs new file mode 100644 index 0000000..edec2ba --- /dev/null +++ b/dotnet/src/HybridRowStress/HybridRowStressProgram.cs @@ -0,0 +1,317 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowStress +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Text; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Microsoft.Extensions.CommandLineUtils; + using Newtonsoft.Json; + + internal class HybridRowStressProgram + { + private static readonly JsonSerializerSettings JsonSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented, + }; + + private readonly HybridRowStressConfig config; + private readonly RandomGenerator rand; + private readonly HybridRowValueGenerator valueGenerator; + private readonly SchemaGenerator schemaGenerator; + + public HybridRowStressProgram(HybridRowStressConfig config) + { + this.config = config; + this.rand = new RandomGenerator(new Random(this.config.Seed)); + this.valueGenerator = new HybridRowValueGenerator(this.rand, config.GeneratorConfig); + this.schemaGenerator = new SchemaGenerator(this.rand, this.config.GeneratorConfig, this.valueGenerator); + } + + public static int Main(string[] args) + { + try + { + HybridRowStressConfig config = HybridRowStressConfig.FromArgs(args); + HybridRowStressProgram program = new HybridRowStressProgram(config); + Task ret = program.MainAsync(); + ret.Wait(); + return ret.Result == 0 ? 0 : 1; + } + catch (CommandParsingException) + { + return -1; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex); + return -1; + } + } + + private static string ToJsonString(Namespace ns) + { + return JsonConvert.SerializeObject(ns, HybridRowStressProgram.JsonSettings); + } + + [SuppressMessage("Microsoft.Reliability", "CA2001:Avoid calling problematic methods", Justification = "Perf Benchmark")] + private Task MainAsync() + { + long totalErrors = 0; + Stopwatch sw = new Stopwatch(); + double jsonWriteMs = 0; + double writeMs = 0; + double streamingMs = 0; + long jsonSize = 0; + long rowSize = 0; + + long iteration; + for (iteration = 0; iteration != this.config.Iterations; iteration += this.config.Iterations < 0 ? 0 : 1) + { + if (this.config.BreakOnError && totalErrors > 0) + { + break; + } + + if ((iteration != 0) && (iteration % this.config.ReportFrequency) == 0) + { + Console.WriteLine( + "{0}: Status: {1} - JSON: {2:F0} Size: {3:F0} JsonWrite: {4:F4} Write: {5:F4} Streaming: {6:F4}", + DateTime.Now, + iteration, + jsonSize / iteration, + rowSize / iteration, + jsonWriteMs / (iteration * this.config.InnerLoopCount), + writeMs / (iteration * this.config.InnerLoopCount), + streamingMs / (iteration * this.config.InnerLoopCount)); + } + + StressContext context = new StressContext(iteration, this.config); + try + { + // Generate a random namespace and table schema. + if (this.config.NamespaceFile != null) + { + string json = File.ReadAllText(this.config.NamespaceFile); + context.Namespace = Namespace.Parse(json); + Contract.Requires(context.Namespace != null); + } + else + { + context.Namespace = this.schemaGenerator.InitializeRandomNamespace(); + } + + if (this.config.TableSchemaName != null) + { + context.TableSchema = context.Namespace.Schemas.Find(s => s.Name == this.config.TableSchemaName); + Contract.Requires(context.TableSchema != null); + } + else + { + context.TableSchema = this.schemaGenerator.InitializeRandomSchema(context.Namespace, 0); + } + + context.IsNotNull(context.Namespace.Schemas.Find(s => s.Name == context.TableSchema.Name), "Namespace contains table schema"); + + if (!HybridRowStressProgram.DoesNamespaceRoundtripToJson(context)) + { + continue; + } + + // Compile the table schema to get a layout. + if (!HybridRowStressProgram.DoesSchemaCompile(context, out Layout layout)) + { + continue; + } + + // Generate a value for writing. + LayoutResolver resolver = new LayoutResolverNamespace(context.Namespace); + int initialCapacity = this.config.GeneratorConfig.RowBufferInitialCapacity.Next(this.rand); + TypeArgument typeArg = new TypeArgument(LayoutType.UDT, new TypeArgumentList(layout.SchemaId)); + context.TableValue = (Dictionary)this.valueGenerator.GenerateLayoutType(resolver, typeArg); + + // Write to JSON + Encoding utf8Encoding = new UTF8Encoding(); + JsonSerializer jsonSerializer = JsonSerializer.Create(HybridRowStressProgram.JsonSettings); + using (MemoryStream jsonStream = new MemoryStream(initialCapacity)) + using (StreamWriter textWriter = new StreamWriter(jsonStream, utf8Encoding)) + using (JsonTextWriter jsonWriter = new JsonTextWriter(textWriter)) + { + try + { + if (this.config.GCBeforeInnerLoop) + { + GC.Collect(); + } + + sw.Restart(); + for (int innerLoop = 0; innerLoop < this.config.InnerLoopCount; innerLoop++) + { + jsonStream.SetLength(0); + jsonSerializer.Serialize(jsonWriter, context.TableValue); + jsonWriter.Flush(); + } + + sw.Stop(); + jsonWriteMs += sw.Elapsed.TotalMilliseconds; + context.Trace("JSON Write: {0,10:F5}", sw.Elapsed.TotalMilliseconds / this.config.InnerLoopCount); + } + catch (Exception ex) + { + context.Fail(ex, "Failed to write row buffer."); + continue; + } + + jsonSize += jsonStream.Length; + context.Trace(JsonConvert.SerializeObject(context.TableValue, HybridRowStressProgram.JsonSettings)); + } + + // Write a row buffer using the layout. + WriteRowGenerator rowGenerator = new WriteRowGenerator(initialCapacity, layout, resolver); + try + { + if (this.config.GCBeforeInnerLoop) + { + GC.Collect(); + } + + sw.Restart(); + for (int innerLoop = 0; innerLoop < this.config.InnerLoopCount; innerLoop++) + { + rowGenerator.Reset(); + + Result r = rowGenerator.DispatchLayout(layout, context.TableValue); + context.IsSuccess(r); + } + + sw.Stop(); + writeMs += sw.Elapsed.TotalMilliseconds; + context.Trace("Patch Write: {0,10:F5}", sw.Elapsed.TotalMilliseconds / this.config.InnerLoopCount); + } + catch (Exception ex) + { + context.Fail(ex, "Failed to write row buffer."); + continue; + } + + // Read the row using a streaming reader. + try + { + rowSize += rowGenerator.Length; + RowReader reader = rowGenerator.GetReader(); + context.Trace(ref reader); + } + catch (Exception ex) + { + context.Fail(ex, "Failed to read row buffer."); + continue; + } + + // Write the same row using the streaming writer. + StreamingRowGenerator writer = new StreamingRowGenerator(initialCapacity, layout, resolver); + try + { + if (this.config.GCBeforeInnerLoop) + { + GC.Collect(); + } + + sw.Restart(); + for (int innerLoop = 0; innerLoop < this.config.InnerLoopCount; innerLoop++) + { + writer.Reset(); + + Result r = writer.WriteBuffer(context.TableValue); + context.IsSuccess(r); + } + + sw.Stop(); + streamingMs += sw.Elapsed.TotalMilliseconds; + context.Trace("Streaming Write: {0,10:F5}", sw.Elapsed.TotalMilliseconds / this.config.InnerLoopCount); + } + catch (Exception ex) + { + context.Fail(ex, "Failed to write streaming row buffer."); + continue; + } + + // Read the row again using a streaming reader. + try + { + Contract.Requires(rowGenerator.Length == writer.Length); + RowReader reader = writer.GetReader(); + context.Trace(ref reader); + } + catch (Exception ex) + { + context.Fail(ex, "Failed to read streaming row buffer."); + } + } + finally + { + totalErrors += context.Errors; + } + } + + Console.WriteLine( + "{0}: Status: {1} : Errors: {2} JSON: {3:F0} Size: {4:F0} JsonWrite: {5:F4} Write: {6:F4} Streaming: {7:F4}", + DateTime.Now, + iteration, + totalErrors, + jsonSize / iteration, + rowSize / iteration, + jsonWriteMs / (iteration * this.config.InnerLoopCount), + writeMs / (iteration * this.config.InnerLoopCount), + streamingMs / (iteration * this.config.InnerLoopCount)); + + return Task.FromResult(totalErrors); + } + + private static bool DoesNamespaceRoundtripToJson(StressContext context) + { + string json1 = HybridRowStressProgram.ToJsonString(context.Namespace); + context.Trace(json1); + + try + { + Namespace ns2 = Namespace.Parse(json1); + string json2 = JsonConvert.SerializeObject(ns2, HybridRowStressProgram.JsonSettings); + return context.AreEqual(json1, json2, "Namespace failed to round-trip."); + } + catch (Exception ex) + { + context.Fail(ex, "Namespace failed to parse with exception."); + return false; + } + } + + private static bool DoesSchemaCompile(StressContext context, out Layout layout) + { + try + { + layout = LayoutCompiler.Compile(context.Namespace, context.TableSchema); + return true; + } + catch (Exception ex) + { + context.Fail(ex, "Namespace failed to compile with exception."); + layout = default; + return false; + } + } + } +} diff --git a/dotnet/src/HybridRowStress/Microsoft.Azure.Cosmos.Serialization.HybridRowStress.csproj b/dotnet/src/HybridRowStress/Microsoft.Azure.Cosmos.Serialization.HybridRowStress.csproj new file mode 100644 index 0000000..22e0096 --- /dev/null +++ b/dotnet/src/HybridRowStress/Microsoft.Azure.Cosmos.Serialization.HybridRowStress.csproj @@ -0,0 +1,25 @@ + + + + true + true + {2FED4096-A113-4946-85E2-36A1A94924C9} + Exe + Test + Microsoft.Azure.Cosmos.Serialization.HybridRowStress + Microsoft.Azure.Cosmos.Serialization.HybridRowStress + netcoreapp2.2 + x64 + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/HybridRowStress/Properties/AssemblyInfo.cs b/dotnet/src/HybridRowStress/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..b5779d0 --- /dev/null +++ b/dotnet/src/HybridRowStress/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRowStress")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("2FED4096-A113-4946-85E2-36A1A94924C9")] diff --git a/dotnet/src/HybridRowStress/StressContext.cs b/dotnet/src/HybridRowStress/StressContext.cs new file mode 100644 index 0000000..48e03b6 --- /dev/null +++ b/dotnet/src/HybridRowStress/StressContext.cs @@ -0,0 +1,142 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowStress +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Core; + using Microsoft.Azure.Cosmos.Core.Utf8; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; + using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas; + using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; + using Newtonsoft.Json; + + public class StressContext + { + private static readonly JsonSerializerSettings JsonSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented, + }; + + public StressContext(long iteration, HybridRowStressConfig config) + { + Contract.Requires(config != null); + + this.Iteration = iteration; + this.Config = config; + } + + public long Iteration { get; } + + public long Errors { get; private set; } + + public HybridRowStressConfig Config { get; } + + public Namespace Namespace { get; set; } + + public Schema TableSchema { get; set; } + + public Dictionary TableValue { get; set; } + + public bool IsSuccess(Result result) + { + if (result != Result.Success) + { + this.ReportFailure(Result.Success, result, string.Empty); + return false; + } + + return true; + } + + public bool AreEqual(T expected, T actual, string msg) + { + if (!actual.Equals(expected)) + { + this.ReportFailure(expected, actual, msg); + return false; + } + + return true; + } + + public bool IsNotNull(object actual, string msg) + { + if (actual == null) + { + this.ReportFailure("not null", "null", msg); + return false; + } + + return true; + } + + public void Fail(Exception ex, string msg) + { + this.Errors++; + this.ReportContext(); + Console.WriteLine("Failure: {0}\n{1}", msg, ex); + } + + public void Trace(ref RowReader reader) + { + if (this.Config.Verbose) + { + Result result = DiagnosticConverter.ReaderToString(ref reader, out string str); + if (result != Result.Success) + { + this.Errors++; + this.ReportContext(); + Console.WriteLine("Trace RowReader Failure: {0}", Enum.GetName(typeof(Result), result)); + return; + } + + Console.WriteLine(str); + } + } + + public void Trace(T arg) + { + if (this.Config.Verbose) + { + Console.WriteLine(arg); + } + } + + public void Trace(string msg) + { + if (this.Config.Verbose) + { + Console.WriteLine(msg); + } + } + + public void Trace(string format, params object[] args) + { + if (this.Config.Verbose) + { + Console.WriteLine(format, args); + } + } + + private void ReportFailure(T expected, T actual, string msg) + { + this.Errors++; + this.ReportContext(); + Console.WriteLine("Failure: expected: '{0}', actual: '{1}' : {2}", expected, actual, msg); + } + + private void ReportContext() + { + Console.WriteLine("Iteration: {0}", this.Iteration); + Console.WriteLine("Config: {0}", JsonConvert.SerializeObject(this.Config, StressContext.JsonSettings)); + Console.WriteLine("Namespace: {0}", JsonConvert.SerializeObject(this.Namespace, StressContext.JsonSettings)); + Console.WriteLine("TableSchema: {0}", this.TableSchema.Name); + Console.WriteLine("TableValue: {0}", JsonConvert.SerializeObject(this.TableValue, StressContext.JsonSettings)); + } + } +} diff --git a/dotnet/src/HybridRowStress/SyntaxException.cs b/dotnet/src/HybridRowStress/SyntaxException.cs new file mode 100644 index 0000000..eecfaed --- /dev/null +++ b/dotnet/src/HybridRowStress/SyntaxException.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Serialization.HybridRowStress +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.Serialization; + + [Serializable] + [ExcludeFromCodeCoverage] + public sealed class SyntaxException : Exception + { + public SyntaxException() + { + } + + public SyntaxException(string message) + : base(message) + { + } + + public SyntaxException(string message, Exception innerException) + : base(message, innerException) + { + } + + private SyntaxException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/dotnet/src/build.props b/dotnet/src/build.props new file mode 100644 index 0000000..1a9cac9 --- /dev/null +++ b/dotnet/src/build.props @@ -0,0 +1,20 @@ + + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + $(MSBuildThisFileDirectory) + + + + + + 1 + 0 + 0 + $(VersionMajor).$(VersionMinor).$(VersionPatch)-preview + $(ProductSemanticVersion) + $(ProductSemanticVersion) + $(ProductSemanticVersion) + + diff --git a/dotnet/src/dirs.proj b/dotnet/src/dirs.proj new file mode 100644 index 0000000..64fd6bc --- /dev/null +++ b/dotnet/src/dirs.proj @@ -0,0 +1,23 @@ + + + + + + + true + + + + + + + + + + + + + + + +