random-mirrors/HybridRow
1
0
Fork 0
mirror of https://github.com/microsoft/HybridRow.git synced 2026-01-24 11:53:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Copied dotnet code from CosmosDB repository

Browse Source
This commit is contained in:
David Noble
2019-08-20 11:58:29 -07:00
parent b0e89b0dda
commit 31f3bc828b
201 changed files with 37803 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
dotnet/src/HybridRowCLI/App.config Normal file
View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" culture="neutral" publicKeyToken="30ad4fe6b2a6aeed" />
<bindingRedirect oldVersion="0.0.0.0-10.0.0.0" newVersion="10.0.0.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

82
dotnet/src/HybridRowCLI/CompileCommand.cs Normal file
View File

@@ -0,0 +1,82 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas;
using Microsoft.Extensions.CommandLineUtils;
public class CompileCommand
{
private CompileCommand()
{
}
public bool Verbose { get; set; } = false;
public List<string> Schemas { get; set; } = null;
public static void AddCommand(CommandLineApplication app)
{
app.Command(
"compile",
command =>
{
command.Description = "Compile and validate a hybrid row schema.";
command.ExtendedHelpText = "Compile a hybrid row schema using the object model compiler and run schema consistency validator.";
command.HelpOption("-? | -h | --help");
CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output. Default: false.", CommandOptionType.NoValue);
CommandArgument schemasOpt = command.Argument(
"schema",
"File(s) containing the schema namespace to compile.",
arg => { arg.MultipleValues = true; });
command.OnExecute(
() =>
{
CompileCommand config = new CompileCommand();
config.Verbose = verboseOpt.HasValue();
config.Schemas = schemasOpt.Values;
return config.OnExecute();
});
});
}
public int OnExecute()
{
foreach (string schemaFile in this.Schemas)
{
if (this.Verbose)
{
Console.WriteLine($"Compiling {schemaFile}...");
Console.WriteLine();
}
string json = File.ReadAllText(schemaFile);
Namespace n = Namespace.Parse(json);
if (this.Verbose)
{
Console.WriteLine($"Namespace: {n.Name}");
foreach (Schema s in n.Schemas)
{
Console.WriteLine($" {s.SchemaId} Schema: {s.Name}");
}
}
if (this.Verbose)
{
Console.WriteLine();
Console.WriteLine($"Compiling {schemaFile} complete.\n");
}
}
return 0;
}
}
}

78
dotnet/src/HybridRowCLI/ConsoleNative.cs Normal file
View File

@@ -0,0 +1,78 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
#pragma warning disable CA1028 // Enum Storage should be Int32
#pragma warning disable CA1707 // Identifiers should not contain underscores
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using System.Runtime.InteropServices;
public static class ConsoleNative
{
[Flags]
public enum ConsoleModes : uint
{
ENABLE_PROCESSED_INPUT = 0x0001,
ENABLE_LINE_INPUT = 0x0002,
ENABLE_ECHO_INPUT = 0x0004,
ENABLE_WINDOW_INPUT = 0x0008,
ENABLE_MOUSE_INPUT = 0x0010,
ENABLE_INSERT_MODE = 0x0020,
ENABLE_QUICK_EDIT_MODE = 0x0040,
ENABLE_EXTENDED_FLAGS = 0x0080,
ENABLE_AUTO_POSITION = 0x0100,
ENABLE_PROCESSED_OUTPUT = 0x0001,
ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002,
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004,
DISABLE_NEWLINE_AUTO_RETURN = 0x0008,
ENABLE_LVB_GRID_WORLDWIDE = 0x0010,
}
private const int StdOutputHandle = -11;
public static ConsoleModes Mode
{
get
{
ConsoleNative.GetConsoleMode(ConsoleNative.GetStdHandle(ConsoleNative.StdOutputHandle), out ConsoleModes mode);
return mode;
}
set => ConsoleNative.SetConsoleMode(ConsoleNative.GetStdHandle(ConsoleNative.StdOutputHandle), value);
}
public static void SetBufferSize(int width, int height)
{
Coord size = new Coord
{
X = (short)width,
Y = (short)height,
};
ConsoleNative.SetConsoleScreenBufferSize(ConsoleNative.GetStdHandle(ConsoleNative.StdOutputHandle), size);
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern IntPtr GetStdHandle(int nStdHandle);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool SetConsoleScreenBufferSize(IntPtr hConsoleHandle, Coord dwSize);
[StructLayout(LayoutKind.Sequential)]
private struct Coord
{
public short X;
public short Y;
}
}
}

474
dotnet/src/HybridRowCLI/Csv2HybridRowCommand.cs Normal file
View File

@@ -0,0 +1,474 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Core;
using Microsoft.Azure.Cosmos.Core.Utf8;
using Microsoft.Azure.Cosmos.Serialization.HybridRow;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO;
using Microsoft.Extensions.CommandLineUtils;
public class Csv2HybridRowCommand
{
private const int InitialCapacity = 2 * 1024 * 1024;
private bool verbose;
private long limit;
private string csvFile;
private string outputFile;
private string namespaceFile;
private string tableName;
private Csv2HybridRowCommand()
{
}
public static void AddCommand(CommandLineApplication app)
{
app.Command(
"csv2hr",
command =>
{
command.Description = "Convert a CSV data set to hybrid row.";
command.ExtendedHelpText =
"Convert textual Comma-Separated-Values data set into a HybridRow.\n\n" +
"The csv2hr command accepts CSV files in the following format:\n" +
"\t* First line contains column names, followed by 0 or more blank rows.\n" +
"\t* All subsequent lines contain rows. A row with too few values treats \n" +
"\t all missing values as the empty string. Lines consistent entirely of \n" +
"\t whitespace are omitted. Columns without a name are discarded.";
command.HelpOption("-? | -h | --help");
CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue);
CommandOption limitOpt = command.Option("--limit", "Limit the number of input rows processed.", CommandOptionType.SingleValue);
CommandOption namespaceOpt = command.Option(
"-n|--namespace",
"File containing the schema namespace.",
CommandOptionType.SingleValue);
CommandOption tableNameOpt = command.Option(
"-tn|--tablename",
"The table schema (when using -namespace). Default: null.",
CommandOptionType.SingleValue);
CommandArgument csvOpt = command.Argument("csv", "File containing csv to convert.");
CommandArgument outputOpt = command.Argument("output", "Output file to contain the HybridRow conversion.");
command.OnExecute(
() =>
{
Csv2HybridRowCommand config = new Csv2HybridRowCommand
{
verbose = verboseOpt.HasValue(),
limit = !limitOpt.HasValue() ? long.MaxValue : long.Parse(limitOpt.Value()),
namespaceFile = namespaceOpt.Value().Trim(),
tableName = tableNameOpt.Value().Trim(),
csvFile = csvOpt.Value.Trim(),
outputFile = outputOpt.Value.Trim()
};
if (string.IsNullOrWhiteSpace(config.csvFile))
{
throw new CommandParsingException(command, "Error: Input file MUST be provided.");
}
if (string.IsNullOrWhiteSpace(config.outputFile))
{
throw new CommandParsingException(command, "Error: Output file MUST be provided.");
}
if (config.csvFile == config.outputFile)
{
throw new CommandParsingException(command, "Error: Input and Output files MUST be different.");
}
return config.OnExecuteAsync().Result;
});
});
}
private static Result ProcessLine(
Utf8String[] paths,
string line,
LayoutResolver resolver,
Layout layout,
MemorySpanResizer<byte> resizer,
out ReadOnlyMemory<byte> record)
{
Contract.Requires(paths != null);
Contract.Requires(line != null);
RowBuffer row = new RowBuffer(resizer.Memory.Length, resizer);
row.InitLayout(HybridRowVersion.V1, layout, resolver);
WriterContext ctx = new WriterContext { Paths = paths, Line = line };
Result r = RowWriter.WriteBuffer(ref row, ctx, Csv2HybridRowCommand.WriteLine);
if (r != Result.Success)
{
record = default;
return r;
}
record = resizer.Memory.Slice(0, row.Length);
return Result.Success;
}
private static Result WriteLine(ref RowWriter writer, TypeArgument typeArg, WriterContext ctx)
{
ReadOnlySpan<char> remaining = ctx.Line.AsSpan();
foreach (Utf8String path in ctx.Paths)
{
int comma = remaining.IndexOf(',');
ReadOnlySpan<char> fieldValue = comma == -1 ? remaining : remaining.Slice(0, comma);
remaining = comma == -1 ? ReadOnlySpan<char>.Empty : remaining.Slice(comma + 1);
Result r = Csv2HybridRowCommand.WriteField(ref writer, path, fieldValue);
if (r != Result.Success)
{
return r;
}
}
return Result.Success;
}
private static Result WriteField(ref RowWriter writer, Utf8String path, ReadOnlySpan<char> fieldValue)
{
writer.Layout.TryFind(path, out LayoutColumn col);
switch (col?.Type?.LayoutCode ?? LayoutCode.Invalid)
{
case LayoutCode.Boolean:
{
if (!bool.TryParse(fieldValue.AsString(), out bool value))
{
goto default;
}
return writer.WriteBool(path, value);
}
case LayoutCode.Int8:
{
if (!sbyte.TryParse(fieldValue.AsString(), out sbyte value))
{
goto default;
}
return writer.WriteInt8(path, value);
}
case LayoutCode.Int16:
{
if (!short.TryParse(fieldValue.AsString(), out short value))
{
goto default;
}
return writer.WriteInt16(path, value);
}
case LayoutCode.Int32:
{
if (!int.TryParse(fieldValue.AsString(), out int value))
{
goto default;
}
return writer.WriteInt32(path, value);
}
case LayoutCode.Int64:
{
if (!long.TryParse(fieldValue.AsString(), out long value))
{
goto default;
}
return writer.WriteInt64(path, value);
}
case LayoutCode.UInt8:
{
if (!byte.TryParse(fieldValue.AsString(), out byte value))
{
goto default;
}
return writer.WriteUInt8(path, value);
}
case LayoutCode.UInt16:
{
if (!ushort.TryParse(fieldValue.AsString(), out ushort value))
{
goto default;
}
return writer.WriteUInt16(path, value);
}
case LayoutCode.UInt32:
{
if (!uint.TryParse(fieldValue.AsString(), out uint value))
{
goto default;
}
return writer.WriteUInt32(path, value);
}
case LayoutCode.UInt64:
{
if (!ulong.TryParse(fieldValue.AsString(), out ulong value))
{
goto default;
}
return writer.WriteUInt64(path, value);
}
case LayoutCode.VarInt:
{
if (!long.TryParse(fieldValue.AsString(), out long value))
{
goto default;
}
return writer.WriteVarInt(path, value);
}
case LayoutCode.VarUInt:
{
if (!ulong.TryParse(fieldValue.AsString(), out ulong value))
{
goto default;
}
return writer.WriteVarUInt(path, value);
}
case LayoutCode.Float32:
{
if (!float.TryParse(fieldValue.AsString(), out float value))
{
goto default;
}
return writer.WriteFloat32(path, value);
}
case LayoutCode.Float64:
{
if (!double.TryParse(fieldValue.AsString(), out double value))
{
goto default;
}
return writer.WriteFloat64(path, value);
}
case LayoutCode.Decimal:
{
if (!decimal.TryParse(fieldValue.AsString(), out decimal value))
{
goto default;
}
return writer.WriteDecimal(path, value);
}
case LayoutCode.DateTime:
{
IFormatProvider provider = CultureInfo.CurrentCulture;
DateTimeStyles style = DateTimeStyles.AdjustToUniversal |
DateTimeStyles.AllowWhiteSpaces |
DateTimeStyles.AssumeUniversal;
if (!DateTime.TryParse(fieldValue.AsString(), provider, style, out DateTime value))
{
goto default;
}
return writer.WriteDateTime(path, value);
}
case LayoutCode.Guid:
{
string s = fieldValue.AsString();
// If the guid is quoted then remove the quotes.
if (s.Length > 2 && s[0] == '"' && s[s.Length - 1] == '"')
{
s = s.Substring(1, s.Length - 2);
}
if (!Guid.TryParse(s, out Guid value))
{
goto default;
}
return writer.WriteGuid(path, value);
}
case LayoutCode.Binary:
{
try
{
byte[] newBytes = Convert.FromBase64String(fieldValue.AsString());
return writer.WriteBinary(path, newBytes.AsSpan());
}
catch (Exception)
{
// fall through and try hex.
}
if (fieldValue.TryParseHexString(out byte[] fromHexBytes))
{
return writer.WriteBinary(path, fromHexBytes);
}
goto default;
}
case LayoutCode.UnixDateTime:
{
if (!long.TryParse(fieldValue.AsString(), out long value))
{
goto default;
}
return writer.WriteUnixDateTime(path, new UnixDateTime(value));
}
case LayoutCode.MongoDbObjectId:
{
string s = fieldValue.AsString();
try
{
byte[] newBytes = Convert.FromBase64String(s);
return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(newBytes.AsSpan()));
}
catch (Exception)
{
// fall through and try hex.
}
if (fieldValue.TryParseHexString(out byte[] fromHexBytes))
{
return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(fromHexBytes));
}
goto default;
}
default:
return writer.WriteString(path, fieldValue.AsString());
}
}
private async Task<int> OnExecuteAsync()
{
LayoutResolver resolver = await SchemaUtil.CreateResolverAsync(this.namespaceFile, this.verbose);
string sdl = string.IsNullOrWhiteSpace(this.namespaceFile) ? null : await File.ReadAllTextAsync(this.namespaceFile);
MemorySpanResizer<byte> resizer = new MemorySpanResizer<byte>(Csv2HybridRowCommand.InitialCapacity);
using (Stream stm = new FileStream(this.csvFile, FileMode.Open))
using (TextReader txt = new StreamReader(stm))
using (Stream outputStm = new FileStream(this.outputFile, FileMode.Create))
{
// Read the CSV "schema".
string fieldNamesLine = await txt.ReadLineAsync();
while (fieldNamesLine != null && string.IsNullOrWhiteSpace(fieldNamesLine))
{
fieldNamesLine = await txt.ReadLineAsync();
}
if (fieldNamesLine == null)
{
await Console.Error.WriteLineAsync($"Input file contains no schema: {this.csvFile}");
return -1;
}
string[] fieldNames = fieldNamesLine.Split(',');
if (!(from c in fieldNames where !string.IsNullOrWhiteSpace(c) select c).Any())
{
await Console.Error.WriteLineAsync($"All columns are ignored. Does this file have field headers?: {this.csvFile}");
return -1;
}
Utf8String[] paths = (from c in fieldNames select Utf8String.TranscodeUtf16(c)).ToArray();
SchemaId tableId = SystemSchema.EmptySchemaId;
if (!string.IsNullOrWhiteSpace(this.tableName))
{
tableId = (resolver as LayoutResolverNamespace)?.Namespace?.Schemas?.Find(s => s.Name == this.tableName)?.SchemaId ??
SchemaId.Invalid;
if (tableId == SchemaId.Invalid)
{
await Console.Error.WriteLineAsync($"Error: schema {this.tableName} could not be found in {this.namespaceFile}.");
return -1;
}
}
Layout layout = resolver.Resolve(tableId);
long totalWritten = 0;
Segment segment = new Segment("CSV conversion from HybridRowCLI csv2hr", sdl);
Result r = await outputStm.WriteRecordIOAsync(
segment,
async index =>
{
if (totalWritten >= this.limit)
{
return (Result.Success, default);
}
string line = await txt.ReadLineAsync();
while (line != null && string.IsNullOrWhiteSpace(line))
{
line = await txt.ReadLineAsync();
}
if (line == null)
{
return (Result.Success, default);
}
Result r2 = Csv2HybridRowCommand.ProcessLine(paths, line, resolver, layout, resizer, out ReadOnlyMemory<byte> record);
if (r2 != Result.Success)
{
return (r2, default);
}
totalWritten++;
return (r2, record);
});
if (r != Result.Success)
{
Console.WriteLine($"Error while writing record: {totalWritten} to HybridRow(s): {this.outputFile}");
return -1;
}
Console.WriteLine($"Wrote ({totalWritten}) HybridRow(s): {this.outputFile}");
return 0;
}
}
private struct WriterContext
{
public Utf8String[] Paths;
public string Line;
}
}
}

41
dotnet/src/HybridRowCLI/HybridRowCLIProgram.cs Normal file
View File

@@ -0,0 +1,41 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using Microsoft.Extensions.CommandLineUtils;
internal class HybridRowCLIProgram
{
private const string Version = "1.0.0";
private const string LongVersion = nameof(HybridRowCLI) + " " + HybridRowCLIProgram.Version;
public static int Main(string[] args)
{
try
{
CommandLineApplication command = new CommandLineApplication(throwOnUnexpectedArg: true)
{
Name = nameof(HybridRowCLI),
Description = "HybridRow Command Line Interface.",
};
command.HelpOption("-? | -h | --help");
command.VersionOption("-ver | --version", HybridRowCLIProgram.Version, HybridRowCLIProgram.LongVersion);
CompileCommand.AddCommand(command);
PrintCommand.AddCommand(command);
Json2HybridRowCommand.AddCommand(command);
Csv2HybridRowCommand.AddCommand(command);
return command.Execute(args);
}
catch (CommandParsingException ex)
{
Console.Error.WriteLine(ex.Message);
return -1;
}
}
}
}

710
dotnet/src/HybridRowCLI/Json2HybridRowCommand.cs Normal file
View File

@@ -0,0 +1,710 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using System.Globalization;
using System.IO;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Core;
using Microsoft.Azure.Cosmos.Serialization.HybridRow;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO;
using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator;
using Microsoft.Extensions.CommandLineUtils;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
public class Json2HybridRowCommand
{
private const int InitialCapacity = 2 * 1024 * 1024;
private Json2HybridRowCommand()
{
}
public bool Verbose { get; set; }
public string JsonFile { get; set; }
public string OutputFile { get; set; }
public string NamespaceFile { get; set; }
public string TableName { get; set; }
public static void AddCommand(CommandLineApplication app)
{
app.Command(
"json2hr",
command =>
{
command.Description = "Convert a JSON document to hybrid row.";
command.ExtendedHelpText =
"Convert textual JSON document into a HybridRow.\n\n" +
"The json2hr command accepts files in two formats:\n" +
"\t* A JSON file whose top-level element is a JSON Object. This file is converted to a HybridRow binary " +
"\t file containing exactly 1 record.\n" +
"\t* A JSON file whose top-level element is a JSON Array. This file is convert to a A HybridRow RecordIO " +
"\t file containing 0 or more records, one for each element of the JSON Array.";
command.HelpOption("-? | -h | --help");
CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue);
CommandOption namespaceOpt = command.Option(
"-n|--namespace",
"File containing the schema namespace.",
CommandOptionType.SingleValue);
CommandOption tableNameOpt = command.Option(
"-tn|--tablename",
"The table schema (when using -namespace). Default: null.",
CommandOptionType.SingleValue);
CommandArgument jsonOpt = command.Argument("json", "File containing json to convert.");
CommandArgument outputOpt = command.Argument("output", "Output file to contain the HybridRow conversion.");
command.OnExecute(
() =>
{
Json2HybridRowCommand config = new Json2HybridRowCommand
{
Verbose = verboseOpt.HasValue(),
NamespaceFile = namespaceOpt.Value(),
TableName = tableNameOpt.Value(),
JsonFile = jsonOpt.Value,
OutputFile = outputOpt.Value
};
return config.OnExecuteAsync().Result;
});
});
}
public async Task<int> OnExecuteAsync()
{
LayoutResolver globalResolver = await SchemaUtil.CreateResolverAsync(this.NamespaceFile, this.Verbose);
string sdl = string.IsNullOrWhiteSpace(this.NamespaceFile) ? null : await File.ReadAllTextAsync(this.NamespaceFile);
MemorySpanResizer<byte> resizer = new MemorySpanResizer<byte>(Json2HybridRowCommand.InitialCapacity);
using (Stream stm = new FileStream(this.JsonFile, FileMode.Open))
using (TextReader txt = new StreamReader(stm))
using (JsonReader reader = new JsonTextReader(txt))
{
// Turn off any special parsing conversions. Just raw JSON tokens.
reader.DateParseHandling = DateParseHandling.None;
reader.FloatParseHandling = FloatParseHandling.Double;
if (!await reader.ReadAsync())
{
await Console.Error.WriteLineAsync("Error: file is empty.");
return -1;
}
switch (reader.TokenType)
{
case JsonToken.StartObject:
{
JObject obj = await JObject.LoadAsync(reader);
Result r = this.ProcessObject(obj, globalResolver, resizer, out ReadOnlyMemory<byte> record);
if (r != Result.Success)
{
Console.WriteLine($"Error while writing record: 0 to HybridRow(s): {this.OutputFile}");
return -1;
}
// Write the output.
using (Stream outputStm = new FileStream(this.OutputFile, FileMode.Create))
{
await outputStm.WriteAsync(record);
Console.WriteLine($"Wrote (1) HybridRow: {this.OutputFile}");
}
return 0;
}
case JsonToken.StartArray:
{
using (Stream outputStm = new FileStream(this.OutputFile, FileMode.Create))
{
long totalWritten = 0;
Segment segment = new Segment("JSON conversion from HybridRowCLI json2hr", sdl);
Result r = await outputStm.WriteRecordIOAsync(
segment,
async index =>
{
if (!await reader.ReadAsync() || reader.TokenType == JsonToken.EndArray)
{
return (Result.Success, default);
}
JObject obj = await JObject.LoadAsync(reader);
Result r2 = this.ProcessObject(obj, globalResolver, resizer, out ReadOnlyMemory<byte> record);
if (r2 != Result.Success)
{
return (r2, default);
}
totalWritten++;
return (r2, record);
});
if (r != Result.Success)
{
Console.WriteLine($"Error while writing record: {totalWritten} to HybridRow(s): {this.OutputFile}");
return -1;
}
Console.WriteLine($"Wrote ({totalWritten}) HybridRow(s): {this.OutputFile}");
}
return 0;
}
default:
await Console.Error.WriteLineAsync("Error: Only JSON documents with top-level Object/Array supported.");
return -1;
}
}
}
private static string TrimPath(JContainer parent, string path)
{
if (string.IsNullOrEmpty(parent.Path))
{
return path;
}
return path.Substring(parent.Path.Length + 1);
}
/// <summary>Returns true if the type code indicates a linear scope (i.e. array like).</summary>
/// <param name="code">The scope type code.</param>
/// </remarks>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsLinearScope(LayoutCode code)
{
if (code < LayoutCode.ObjectScope || code >= LayoutCode.EndScope)
{
return false;
}
const ulong bitmask =
(0x1UL << (int)(LayoutCode.ArrayScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableArrayScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.TypedArrayScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableTypedArrayScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.TypedSetScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableTypedSetScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.TypedMapScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableTypedMapScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.TypedTupleScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableTypedTupleScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.TaggedScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableTaggedScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.Tagged2Scope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableTagged2Scope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.NullableScope - LayoutCode.ObjectScope)) |
(0x1UL << (int)(LayoutCode.ImmutableNullableScope - LayoutCode.ObjectScope))
;
return ((0x1UL << (code - LayoutCode.ObjectScope)) & bitmask) != 0;
}
private Result ProcessObject(JObject obj, LayoutResolver resolver, MemorySpanResizer<byte> resizer, out ReadOnlyMemory<byte> record)
{
Contract.Requires(obj.Type == JTokenType.Object);
SchemaId tableId = SystemSchema.EmptySchemaId;
if (!string.IsNullOrWhiteSpace(this.TableName))
{
tableId = (resolver as LayoutResolverNamespace)?.Namespace?.Schemas?.Find(s => s.Name == this.TableName)?.SchemaId ??
SchemaId.Invalid;
if (tableId == SchemaId.Invalid)
{
Console.Error.WriteLine($"Error: schema {this.TableName} could not be found in {this.NamespaceFile}.");
record = default;
return Result.Failure;
}
}
Layout layout = resolver.Resolve(tableId);
RowBuffer row = new RowBuffer(resizer.Memory.Length, resizer);
row.InitLayout(HybridRowVersion.V1, layout, resolver);
WriterContext ctx = new WriterContext { Token = obj, PathPrefix = "" };
Result r = RowWriter.WriteBuffer(ref row, ctx, this.WriteObject);
if (r != Result.Success)
{
record = default;
return r;
}
record = resizer.Memory.Slice(0, row.Length);
return Result.Success;
}
private Result WriteArray(ref RowWriter writer, TypeArgument typeArg, WriterContext ctx)
{
JArray obj = ctx.Token as JArray;
TypeArgument propTypeArgument = typeArg.TypeArgs.Count > 0 ? typeArg.TypeArgs[0] : default;
foreach (JToken token in obj.Children())
{
Result r = this.WriteJToken(ref writer, ctx.PathPrefix, null, propTypeArgument, token);
if (r != Result.Success)
{
return r;
}
}
return Result.Success;
}
private Result WriteObject(ref RowWriter writer, TypeArgument typeArg, WriterContext ctx)
{
JObject obj = ctx.Token as JObject;
Layout layout = writer.Layout;
if (typeArg.TypeArgs.SchemaId != SchemaId.Invalid)
{
layout = writer.Resolver.Resolve(typeArg.TypeArgs.SchemaId);
ctx.PathPrefix = "";
}
foreach (JProperty p in obj.Properties())
{
if (!p.HasValues)
{
continue;
}
string path = Json2HybridRowCommand.TrimPath(obj, p.Path);
TypeArgument propTypeArg = default;
string propFullPath = ctx.PathPrefix + path;
if (layout.TryFind(propFullPath, out LayoutColumn col))
{
propTypeArg = col.TypeArg;
}
Result r = this.WriteJToken(ref writer, ctx.PathPrefix, path, propTypeArg, p.Value);
if (r != Result.Success)
{
return r;
}
}
return Result.Success;
}
private Result WriteJToken(ref RowWriter writer, string pathPrefix, string path, TypeArgument typeArg, JToken token)
{
TypeArgument propTypeArg;
LayoutCode code = typeArg.Type?.LayoutCode ?? LayoutCode.Invalid;
switch (token.Type)
{
case JTokenType.Object:
propTypeArg = typeArg.Type is LayoutUDT ? typeArg : new TypeArgument(LayoutType.Object);
WriterContext ctx1 = new WriterContext { Token = token.Value<JObject>(), PathPrefix = $"{pathPrefix}{path}." };
return writer.WriteScope(path, propTypeArg, ctx1, this.WriteObject);
case JTokenType.Array:
propTypeArg = Json2HybridRowCommand.IsLinearScope(typeArg.Type?.LayoutCode ?? LayoutCode.Invalid)
? typeArg
: new TypeArgument(LayoutType.Array);
WriterContext ctx2 = new WriterContext { Token = token.Value<JArray>(), PathPrefix = $"{pathPrefix}{path}[]." };
return writer.WriteScope(path, propTypeArg, ctx2, this.WriteArray);
case JTokenType.Integer:
{
long value = token.Value<long>();
try
{
switch (code)
{
case LayoutCode.Int8:
return writer.WriteInt8(path, (sbyte)value);
case LayoutCode.Int16:
return writer.WriteInt16(path, (short)value);
case LayoutCode.Int32:
return writer.WriteInt32(path, (int)value);
case LayoutCode.Int64:
return writer.WriteInt64(path, value);
case LayoutCode.UInt8:
return writer.WriteUInt8(path, (byte)value);
case LayoutCode.UInt16:
return writer.WriteUInt16(path, (ushort)value);
case LayoutCode.UInt32:
return writer.WriteUInt32(path, (uint)value);
case LayoutCode.UInt64:
return writer.WriteUInt64(path, (ulong)value);
case LayoutCode.VarInt:
return writer.WriteVarInt(path, value);
case LayoutCode.VarUInt:
return writer.WriteVarUInt(path, (ulong)value);
case LayoutCode.Float32:
return writer.WriteFloat32(path, value);
case LayoutCode.Float64:
return writer.WriteFloat64(path, value);
case LayoutCode.Decimal:
return writer.WriteDecimal(path, value);
case LayoutCode.Utf8:
return writer.WriteString(path, value.ToString());
}
}
catch (OverflowException)
{
// Ignore overflow, and just write value as a long.
}
return writer.WriteInt64(path, value);
}
case JTokenType.Float:
{
double value = token.Value<double>();
try
{
switch (code)
{
case LayoutCode.Int8:
return writer.WriteInt8(path, (sbyte)value);
case LayoutCode.Int16:
return writer.WriteInt16(path, (short)value);
case LayoutCode.Int32:
return writer.WriteInt32(path, (int)value);
case LayoutCode.Int64:
return writer.WriteInt64(path, (long)value);
case LayoutCode.UInt8:
return writer.WriteUInt8(path, (byte)value);
case LayoutCode.UInt16:
return writer.WriteUInt16(path, (ushort)value);
case LayoutCode.UInt32:
return writer.WriteUInt32(path, (uint)value);
case LayoutCode.UInt64:
return writer.WriteUInt64(path, (ulong)value);
case LayoutCode.VarInt:
return writer.WriteVarInt(path, (long)value);
case LayoutCode.VarUInt:
return writer.WriteVarUInt(path, (ulong)value);
case LayoutCode.Float32:
return writer.WriteFloat32(path, (float)value);
case LayoutCode.Float64:
return writer.WriteFloat64(path, value);
case LayoutCode.Decimal:
return writer.WriteDecimal(path, (decimal)value);
case LayoutCode.Utf8:
return writer.WriteString(path, value.ToString(CultureInfo.InvariantCulture));
}
}
catch (OverflowException)
{
// Ignore overflow, and just write value as a double.
}
return writer.WriteFloat64(path, value);
}
case JTokenType.String:
switch (code)
{
case LayoutCode.Boolean:
{
if (!bool.TryParse(token.Value<string>(), out bool value))
{
goto default;
}
return writer.WriteBool(path, value);
}
case LayoutCode.Int8:
{
if (!sbyte.TryParse(token.Value<string>(), out sbyte value))
{
goto default;
}
return writer.WriteInt8(path, value);
}
case LayoutCode.Int16:
{
if (!short.TryParse(token.Value<string>(), out short value))
{
goto default;
}
return writer.WriteInt16(path, value);
}
case LayoutCode.Int32:
{
if (!int.TryParse(token.Value<string>(), out int value))
{
goto default;
}
return writer.WriteInt32(path, value);
}
case LayoutCode.Int64:
{
if (!long.TryParse(token.Value<string>(), out long value))
{
goto default;
}
return writer.WriteInt64(path, value);
}
case LayoutCode.UInt8:
{
if (!byte.TryParse(token.Value<string>(), out byte value))
{
goto default;
}
return writer.WriteUInt8(path, value);
}
case LayoutCode.UInt16:
{
if (!ushort.TryParse(token.Value<string>(), out ushort value))
{
goto default;
}
return writer.WriteUInt16(path, value);
}
case LayoutCode.UInt32:
{
if (!uint.TryParse(token.Value<string>(), out uint value))
{
goto default;
}
return writer.WriteUInt32(path, value);
}
case LayoutCode.UInt64:
{
if (!ulong.TryParse(token.Value<string>(), out ulong value))
{
goto default;
}
return writer.WriteUInt64(path, value);
}
case LayoutCode.VarInt:
{
if (!long.TryParse(token.Value<string>(), out long value))
{
goto default;
}
return writer.WriteVarInt(path, value);
}
case LayoutCode.VarUInt:
{
if (!ulong.TryParse(token.Value<string>(), out ulong value))
{
goto default;
}
return writer.WriteVarUInt(path, value);
}
case LayoutCode.Float32:
{
if (!float.TryParse(token.Value<string>(), out float value))
{
goto default;
}
return writer.WriteFloat32(path, value);
}
case LayoutCode.Float64:
{
if (!double.TryParse(token.Value<string>(), out double value))
{
goto default;
}
return writer.WriteFloat64(path, value);
}
case LayoutCode.Decimal:
{
if (!decimal.TryParse(token.Value<string>(), out decimal value))
{
goto default;
}
return writer.WriteDecimal(path, value);
}
case LayoutCode.DateTime:
{
IFormatProvider provider = CultureInfo.CurrentCulture;
DateTimeStyles style = DateTimeStyles.AdjustToUniversal |
DateTimeStyles.AllowWhiteSpaces |
DateTimeStyles.AssumeUniversal;
if (!DateTime.TryParse(token.Value<string>(), provider, style, out DateTime value))
{
goto default;
}
return writer.WriteDateTime(path, value);
}
case LayoutCode.Guid:
{
string s = token.Value<string>();
// If the guid is quoted then remove the quotes.
if (s.Length > 2 && s[0] == '"' && s[s.Length - 1] == '"')
{
s = s.Substring(1, s.Length - 2);
}
if (!Guid.TryParse(s, out Guid value))
{
goto default;
}
return writer.WriteGuid(path, value);
}
case LayoutCode.Binary:
{
string s = token.Value<string>();
try
{
byte[] newBytes = Convert.FromBase64String(s);
return writer.WriteBinary(path, newBytes.AsSpan());
}
catch (Exception)
{
// fall through and try hex.
}
try
{
byte[] newBytes = ByteConverter.ToBytes(s);
return writer.WriteBinary(path, newBytes.AsSpan());
}
catch (Exception)
{
goto default;
}
}
case LayoutCode.UnixDateTime:
{
if (!long.TryParse(token.Value<string>(), out long value))
{
goto default;
}
return writer.WriteUnixDateTime(path, new UnixDateTime(value));
}
case LayoutCode.MongoDbObjectId:
{
string s = token.Value<string>();
try
{
byte[] newBytes = Convert.FromBase64String(s);
return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(newBytes.AsSpan()));
}
catch (Exception)
{
// fall through and try hex.
}
try
{
byte[] newBytes = ByteConverter.ToBytes(s);
return writer.WriteMongoDbObjectId(path, new MongoDbObjectId(newBytes.AsSpan()));
}
catch (Exception)
{
goto default;
}
}
default:
return writer.WriteString(path, token.Value<string>());
}
case JTokenType.Boolean:
{
bool value = token.Value<bool>();
switch (code)
{
case LayoutCode.Int8:
return writer.WriteInt8(path, (sbyte)(value ? 1 : 0));
case LayoutCode.Int16:
return writer.WriteInt16(path, (short)(value ? 1 : 0));
case LayoutCode.Int32:
return writer.WriteInt32(path, value ? 1 : 0);
case LayoutCode.Int64:
return writer.WriteInt64(path, value ? 1L : 0L);
case LayoutCode.UInt8:
return writer.WriteUInt8(path, (byte)(value ? 1 : 0));
case LayoutCode.UInt16:
return writer.WriteUInt16(path, (ushort)(value ? 1 : 0));
case LayoutCode.UInt32:
return writer.WriteUInt32(path, (uint)(value ? 1 : 0));
case LayoutCode.UInt64:
return writer.WriteUInt64(path, value ? 1U : 0U);
case LayoutCode.VarInt:
return writer.WriteVarInt(path, value ? 1L : 0L);
case LayoutCode.VarUInt:
return writer.WriteVarUInt(path, value ? 1U : 0U);
case LayoutCode.Float32:
return writer.WriteFloat32(path, value ? 1 : 0);
case LayoutCode.Float64:
return writer.WriteFloat64(path, value ? 1 : 0);
case LayoutCode.Decimal:
return writer.WriteDecimal(path, value ? 1 : 0);
case LayoutCode.Utf8:
return writer.WriteString(path, value ? "true" : "false");
default:
return writer.WriteBool(path, value);
}
}
case JTokenType.Null:
switch (code)
{
case LayoutCode.Null:
case LayoutCode.Invalid:
return writer.WriteNull(path);
default:
// Any other schematized type then just don't write it.
return Result.Success;
}
case JTokenType.Comment:
return Result.Success;
default:
Console.Error.WriteLine($"Error: Unexpected JSON token type: {token.Type}.");
return Result.InvalidRow;
}
}
private struct WriterContext
{
public JToken Token;
public string PathPrefix;
}
}
}

30
dotnet/src/HybridRowCLI/Microsoft.Azure.Cosmos.Serialization.HybridRowCLI.csproj Normal file
View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<CheckForOverflowUnderflow>true</CheckForOverflowUnderflow>
<ProjectGuid>{F7D04E9B-4257-4D7E-9AAD-C743AEDBED04}</ProjectGuid>
<OutputType>Exe</OutputType>
<SigningType>Test</SigningType>
<RootNamespace>Microsoft.Azure.Cosmos.Serialization.HybridRowCLI</RootNamespace>
<AssemblyName>Microsoft.Azure.Cosmos.Serialization.HybridRowCLI</AssemblyName>
<TargetFramework>netcoreapp2.2</TargetFramework>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Build.props))\build.props" />
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" />
<PackageReference Include="System.Memory" />
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" />
<PackageReference Include="System.Threading.Tasks.Extensions" />
<PackageReference Include="System.ValueTuple" />
<PackageReference Include="MongoDB.Bson.Signed" />
<PackageReference Include="Microsoft.Extensions.CommandLineUtils" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Core\Core\Microsoft.Azure.Cosmos.Core.csproj" />
<ProjectReference Include="..\HybridRow.Json\Microsoft.Azure.Cosmos.Serialization.HybridRow.Json.csproj" />
<ProjectReference Include="..\HybridRowGenerator\Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator.csproj" />
<ProjectReference Include="..\HybridRow\Microsoft.Azure.Cosmos.Serialization.HybridRow.csproj" />
</ItemGroup>
</Project>

441
dotnet/src/HybridRowCLI/PrintCommand.cs Normal file
View File

@@ -0,0 +1,441 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Core;
using Microsoft.Azure.Cosmos.Serialization.HybridRow;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Json;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO;
using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator;
using Microsoft.Extensions.CommandLineUtils;
public class PrintCommand
{
private const int InitialCapacity = 2 * 1024 * 1024;
private bool showSchema;
private bool verbose;
private bool interactive;
private bool outputJson;
private char outputJsonQuote;
private string outputJsonIndent;
private string namespaceFile;
private List<string> rows;
private PrintCommand()
{
}
public static void AddCommand(CommandLineApplication app)
{
app.Command(
"print",
command =>
{
command.Description = "Print hybrid row value(s).";
command.ExtendedHelpText =
"Convert binary serialized hybrid row value(s) to a printable string form using the given schema.\n\n" +
"The print command accepts files in three formats:\n" +
"\t* A HybridRow RecordIO file containing 0 or more records (with or without embedded SDL).\n" +
"\t* A HybridRow binary file containing exactly 1 record. The length of the file indicates the record size.\n" +
"\t* A HybridRow text file containing exactly 1 record written as a HEX text string. The length of the file\n" +
"\t indicates the length of the encoding. All non-HEX characters (e.g. extra spaces, newlines, or dashes)\n" +
"\t are ignored when decoding the file from HEX string to binary.";
command.HelpOption("-? | -h | --help");
CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue);
CommandOption showSchemaOpt = command.Option(
"--show-schema",
"Include embedded the schema in the output.",
CommandOptionType.NoValue);
CommandOption interactiveOpt = command.Option("-i|--interactive", "Use interactive interface.", CommandOptionType.NoValue);
CommandOption jsonOpt = command.Option("-j|--json", "Output in JSON.", CommandOptionType.NoValue);
CommandOption jsonSingleQuoteOpt = command.Option("--json-single-quote", "Use single quotes in JSON.", CommandOptionType.NoValue);
CommandOption jsonNoIndentOpt = command.Option("--json-no-indent", "Don't indent in JSON.", CommandOptionType.NoValue);
CommandOption namespaceOpt = command.Option(
"-n|--namespace",
"File containing the schema namespace.",
CommandOptionType.SingleValue);
CommandArgument rowsOpt = command.Argument(
"rows",
"File(s) containing rows to print.",
arg => { arg.MultipleValues = true; });
command.OnExecute(
() =>
{
PrintCommand config = new PrintCommand
{
verbose = verboseOpt.HasValue(),
showSchema = showSchemaOpt.HasValue(),
outputJson = jsonOpt.HasValue(),
outputJsonQuote = jsonSingleQuoteOpt.HasValue() ? '\'' : '"',
outputJsonIndent = jsonNoIndentOpt.HasValue() ? null : " ",
interactive = interactiveOpt.HasValue(),
namespaceFile = namespaceOpt.Value(),
rows = rowsOpt.Values
};
return config.OnExecuteAsync().Result;
});
});
}
/// <summary>
/// Read a <see cref="HybridRowHeader" /> from the current stream position without moving the
/// position.
/// </summary>
/// <param name="stm">The stream to read from.</param>
/// <returns>The header read at the current position.</returns>
/// <exception cref="Exception">
/// If a header cannot be read from the current stream position for any
/// reason.
/// </exception>
/// <remarks>
/// On success the stream's position is not changed. On error the stream position is
/// undefined.
/// </remarks>
private static async Task<HybridRowHeader> PeekHybridRowHeaderAsync(Stream stm)
{
try
{
Memory<byte> bytes = await PrintCommand.ReadFixedAsync(stm, HybridRowHeader.Size);
return MemoryMarshal.Read<HybridRowHeader>(bytes.Span);
}
finally
{
stm.Seek(-HybridRowHeader.Size, SeekOrigin.Current);
}
}
/// <summary>Reads a fixed length segment from a stream.</summary>
/// <param name="stm">The stream to read from.</param>
/// <param name="length">The length to read in bytes.</param>
/// <returns>A sequence of bytes read from the stream exactly <paramref name="length" /> long.</returns>
/// <exception cref="Exception">
/// if <paramref name="length" /> bytes cannot be read from the current
/// stream position.
/// </exception>
private static async Task<Memory<byte>> ReadFixedAsync(Stream stm, int length)
{
Memory<byte> bytes = new byte[length].AsMemory();
Memory<byte> active = bytes;
int bytesRead;
do
{
bytesRead = await stm.ReadAsync(active);
active = active.Slice(bytesRead);
}
while (bytesRead != 0);
if (active.Length != 0)
{
throw new Exception("Failed to parse row header");
}
return bytes;
}
private static void Refresh(ConsoleNative.ConsoleModes origMode, Layout layout, int length, long index, string str)
{
string[] lines = str.Split('\n');
int height = lines.Length + 2;
int width = (from line in lines select line.Length).Max();
height = Math.Max(height, Console.WindowHeight);
width = Math.Max(width, Console.WindowWidth);
ConsoleNative.Mode = origMode & ~ConsoleNative.ConsoleModes.ENABLE_WRAP_AT_EOL_OUTPUT;
Console.CursorVisible = false;
ConsoleNative.SetBufferSize(width, height);
Console.Clear();
if (layout != null)
{
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine($"[{index}] Schema: {layout.SchemaId} {layout.Name}, Length: {length}");
}
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine(str);
Console.SetWindowPosition(0, 0);
}
private static bool ShowInteractive(Layout layout, int length, long index, string str)
{
int origBufferHeight = Console.BufferHeight;
int origBufferWidth = Console.BufferWidth;
ConsoleNative.ConsoleModes origMode = ConsoleNative.Mode;
try
{
PrintCommand.Refresh(origMode, layout, length, index, str);
while (true)
{
ConsoleKeyInfo cki = Console.ReadKey(true);
if ((cki.Modifiers & ConsoleModifiers.Alt) != 0 ||
(cki.Modifiers & ConsoleModifiers.Shift) != 0 ||
(cki.Modifiers & ConsoleModifiers.Control) != 0)
{
continue;
}
int top;
int left;
switch (cki.Key)
{
case ConsoleKey.Q:
case ConsoleKey.Escape:
return false;
case ConsoleKey.Spacebar:
case ConsoleKey.N:
case ConsoleKey.Enter:
return true;
case ConsoleKey.R:
PrintCommand.Refresh(origMode, layout, length, index, str);
break;
case ConsoleKey.Home:
Console.SetWindowPosition(0, 0);
break;
case ConsoleKey.End:
Console.SetWindowPosition(Console.WindowLeft, Console.BufferHeight - Console.WindowHeight);
break;
case ConsoleKey.PageDown:
top = Console.WindowTop + Console.WindowHeight;
top = Math.Min(top, Console.BufferHeight - Console.WindowHeight);
Console.SetWindowPosition(Console.WindowLeft, top);
break;
case ConsoleKey.PageUp:
top = Console.WindowTop - Console.WindowHeight;
top = Math.Max(top, 0);
Console.SetWindowPosition(Console.WindowLeft, top);
break;
case ConsoleKey.DownArrow:
top = Console.WindowTop + 1;
top = Math.Min(top, Console.BufferHeight - Console.WindowHeight);
Console.SetWindowPosition(Console.WindowLeft, top);
break;
case ConsoleKey.UpArrow:
top = Console.WindowTop - 1;
top = Math.Max(top, 0);
Console.SetWindowPosition(Console.WindowLeft, top);
break;
case ConsoleKey.RightArrow:
left = Console.WindowLeft + 1;
left = Math.Min(left, Console.BufferWidth - Console.WindowWidth);
Console.SetWindowPosition(left, Console.WindowTop);
break;
case ConsoleKey.LeftArrow:
left = Console.WindowLeft - 1;
left = Math.Max(left, 0);
Console.SetWindowPosition(left, Console.WindowTop);
break;
}
}
}
finally
{
ConsoleNative.Mode = origMode;
Console.ResetColor();
origBufferWidth = Math.Max(Console.WindowWidth, origBufferWidth);
origBufferHeight = Math.Max(Console.WindowHeight, origBufferHeight);
Console.SetBufferSize(origBufferWidth, origBufferHeight);
Console.Clear();
Console.CursorVisible = true;
}
}
/// <summary>Convert a text stream containing a sequence of HEX characters into a byte sequence.</summary>
/// <param name="stm">The stream to read the text from.</param>
/// <returns>The entire contents of the stream interpreted as HEX characters.</returns>
/// <remarks>Any character that is not a HEX character is ignored.</remarks>
private static async Task<Memory<byte>> HexTextStreamToBinaryAsync(Stream stm)
{
using (TextReader reader = new StreamReader(stm))
{
string rowAsText = await reader.ReadToEndAsync();
string trimmed = string.Concat(
from c in rowAsText
where (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
select c);
return ByteConverter.ToBytes(trimmed).AsMemory();
}
}
private async Task<int> OnExecuteAsync()
{
LayoutResolver globalResolver = await SchemaUtil.CreateResolverAsync(this.namespaceFile, this.verbose);
MemorySpanResizer<byte> resizer = new MemorySpanResizer<byte>(PrintCommand.InitialCapacity);
foreach (string rowFile in this.rows)
{
using (Stream stm = new FileStream(rowFile, FileMode.Open))
{
// Detect if it is a text or binary file via the encoding of the magic number at the
// beginning of the HybridRow header.
int magicNumber = stm.ReadByte();
stm.Seek(-1, SeekOrigin.Current);
if (magicNumber == -1)
{
continue; // empty file
}
Result r = Result.Failure;
if (magicNumber == (int)HybridRowVersion.V1)
{
HybridRowHeader header = await PrintCommand.PeekHybridRowHeaderAsync(stm);
if (header.SchemaId == SystemSchema.SegmentSchemaId)
{
r = await this.PrintRecordIOAsync(stm, globalResolver, resizer);
}
else
{
Memory<byte> rowAsBinary = await PrintCommand.ReadFixedAsync(stm, (int)stm.Length);
r = this.PrintOneRow(rowAsBinary, 0, globalResolver);
}
}
else if (char.ToUpper((char)magicNumber, CultureInfo.InvariantCulture) == HybridRowVersion.V1.ToString("X")[0])
{
// Convert hex text file to binary.
Memory<byte> rowAsBinary = await PrintCommand.HexTextStreamToBinaryAsync(stm);
r = this.PrintOneRow(rowAsBinary, 0, globalResolver);
}
if (r == Result.Canceled)
{
return 0;
}
if (r != Result.Success)
{
await Console.Error.WriteLineAsync($"Error reading row at {rowFile}");
return -1;
}
}
}
return 0;
}
/// <summary>Print all records from a HybridRow RecordIO file.</summary>
/// <param name="stm">Stream containing the HybridRow RecordIO content.</param>
/// <param name="globalResolver">The global resolver to use when segments don't contain embedded SDL.</param>
/// <param name="resizer">The resizer for allocating row buffers.</param>
/// <returns>Success if the print is successful, an error code otherwise.</returns>
private async Task<Result> PrintRecordIOAsync(Stream stm, LayoutResolver globalResolver, MemorySpanResizer<byte> resizer)
{
LayoutResolver segmentResolver = globalResolver;
// Read a RecordIO stream.
long index = 0;
Result r = await stm.ReadRecordIOAsync(
record => this.PrintOneRow(record, index++, segmentResolver),
segment =>
{
r = SegmentSerializer.Read(segment.Span, globalResolver, out Segment s);
if (r != Result.Success)
{
return r;
}
segmentResolver = string.IsNullOrWhiteSpace(s.SDL) ? globalResolver : SchemaUtil.LoadFromSdl(s.SDL, this.verbose, globalResolver);
if (this.showSchema)
{
string str = string.IsNullOrWhiteSpace(s.SDL) ? "<empty>" : s.SDL;
if (!this.interactive)
{
Console.WriteLine(str);
}
else
{
if (!PrintCommand.ShowInteractive(null, 0, -1, str))
{
return Result.Canceled;
}
}
}
return Result.Success;
},
resizer);
return r;
}
/// <summary>Print a single HybridRow record.</summary>
/// <param name="buffer">The raw bytes of the row.</param>
/// <param name="index">
/// A 0-based index of the row relative to its outer container (for display
/// purposes only).
/// </param>
/// <param name="resolver">The resolver for nested contents within the row.</param>
/// <returns>Success if the print is successful, an error code otherwise.</returns>
private Result PrintOneRow(Memory<byte> buffer, long index, LayoutResolver resolver)
{
RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, resolver);
RowReader reader = new RowReader(ref row);
Result r;
string str;
if (this.outputJson)
{
r = reader.ToJson(new RowReaderJsonSettings(this.outputJsonIndent, this.outputJsonQuote), out str);
}
else
{
r = DiagnosticConverter.ReaderToString(ref reader, out str);
}
if (r != Result.Success)
{
return r;
}
Layout layout = resolver.Resolve(row.Header.SchemaId);
if (!this.interactive)
{
Console.WriteLine($"Schema: {layout.SchemaId} {layout.Name}, Length: {row.Length}");
Console.WriteLine(str);
}
else
{
if (!PrintCommand.ShowInteractive(layout, row.Length, index, str))
{
return Result.Canceled;
}
}
return Result.Success;
}
}
}

19
dotnet/src/HybridRowCLI/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,19 @@
// <copyright file="AssemblyInfo.cs" company="Microsoft">
// Copyright (c) Microsoft. All rights reserved.
// </copyright>
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Microsoft.Azure.Cosmos.Serialization.HybridRowCLI")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("F7D04E9B-4257-4D7E-9AAD-C743AEDBED04")]

72
dotnet/src/HybridRowCLI/SchemaUtil.cs Normal file
View File

@@ -0,0 +1,72 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Core;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts;
using Microsoft.Azure.Cosmos.Serialization.HybridRow.Schemas;
public static class SchemaUtil
{
/// <summary>Create a resolver.</summary>
/// <param name="namespaceFile">
/// Optional namespace file containing a namespace to be included in the
/// resolver.
/// </param>
/// <param name="verbose">True if verbose output should be written to stdout.</param>
/// <returns>A resolver.</returns>
public static async Task<LayoutResolver> CreateResolverAsync(string namespaceFile, bool verbose)
{
LayoutResolver globalResolver;
if (string.IsNullOrWhiteSpace(namespaceFile))
{
globalResolver = SystemSchema.LayoutResolver;
}
else
{
if (verbose)
{
Console.WriteLine($"Loading {namespaceFile}...");
Console.WriteLine();
}
string json = await File.ReadAllTextAsync(namespaceFile);
globalResolver = SchemaUtil.LoadFromSdl(json, verbose, SystemSchema.LayoutResolver);
}
Contract.Requires(globalResolver != null);
return globalResolver;
}
/// <summary>Create a HybridRow resolver for given piece of embedded Schema Definition Language (SDL).</summary>
/// <param name="json">The SDL to parse.</param>
/// <param name="verbose">True if verbose output should be written to stdout.</param>
/// <returns>A resolver that resolves all types in the given SDL.</returns>
public static LayoutResolver LoadFromSdl(string json, bool verbose, LayoutResolver parent = default)
{
Namespace n = Namespace.Parse(json);
if (verbose)
{
Console.WriteLine($"Namespace: {n.Name}");
foreach (Schema s in n.Schemas)
{
Console.WriteLine($" {s.SchemaId} Schema: {s.Name}");
}
}
LayoutResolver resolver = new LayoutResolverNamespace(n, parent);
if (verbose)
{
Console.WriteLine();
Console.WriteLine($"Loaded {n.Name}.\n");
}
return resolver;
}
}
}

19
dotnet/src/HybridRowCLI/StringExtensions.cs Normal file
View File

@@ -0,0 +1,19 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI
{
using System;
// TODO: this class should go away once we move to .NET Core 2.1.
internal static class StringExtensions
{
public static unsafe string AsString(this ReadOnlySpan<char> span)
{
fixed (char* p = span)
{
return new string(p, 0, span.Length);
}
}
}
}