// ------------------------------------------------------------ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ namespace Microsoft.Azure.Cosmos.Serialization.HybridRowCLI { using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Core; using Microsoft.Azure.Cosmos.Serialization.HybridRow; using Microsoft.Azure.Cosmos.Serialization.HybridRow.IO; using Microsoft.Azure.Cosmos.Serialization.HybridRow.Json; using Microsoft.Azure.Cosmos.Serialization.HybridRow.Layouts; using Microsoft.Azure.Cosmos.Serialization.HybridRow.RecordIO; using Microsoft.Azure.Cosmos.Serialization.HybridRowGenerator; using Microsoft.Extensions.CommandLineUtils; public class PrintCommand { private const int InitialCapacity = 2 * 1024 * 1024; private bool showSchema; private bool verbose; private bool interactive; private bool outputJson; private char outputJsonQuote; private string outputJsonIndent; private string namespaceFile; private List rows; private PrintCommand() { } public static void AddCommand(CommandLineApplication app) { app.Command( "print", command => { command.Description = "Print hybrid row value(s)."; command.ExtendedHelpText = "Convert binary serialized hybrid row value(s) to a printable string form using the given schema.\n\n" + "The print command accepts files in three formats:\n" + "\t* A HybridRow RecordIO file containing 0 or more records (with or without embedded SDL).\n" + "\t* A HybridRow binary file containing exactly 1 record. The length of the file indicates the record size.\n" + "\t* A HybridRow text file containing exactly 1 record written as a HEX text string. The length of the file\n" + "\t indicates the length of the encoding. All non-HEX characters (e.g. extra spaces, newlines, or dashes)\n" + "\t are ignored when decoding the file from HEX string to binary."; command.HelpOption("-? | -h | --help"); CommandOption verboseOpt = command.Option("-v|--verbose", "Display verbose output.", CommandOptionType.NoValue); CommandOption showSchemaOpt = command.Option( "--show-schema", "Include embedded the schema in the output.", CommandOptionType.NoValue); CommandOption interactiveOpt = command.Option("-i|--interactive", "Use interactive interface.", CommandOptionType.NoValue); CommandOption jsonOpt = command.Option("-j|--json", "Output in JSON.", CommandOptionType.NoValue); CommandOption jsonSingleQuoteOpt = command.Option("--json-single-quote", "Use single quotes in JSON.", CommandOptionType.NoValue); CommandOption jsonNoIndentOpt = command.Option("--json-no-indent", "Don't indent in JSON.", CommandOptionType.NoValue); CommandOption namespaceOpt = command.Option( "-n|--namespace", "File containing the schema namespace.", CommandOptionType.SingleValue); CommandArgument rowsOpt = command.Argument( "rows", "File(s) containing rows to print.", arg => { arg.MultipleValues = true; }); command.OnExecute( () => { PrintCommand config = new PrintCommand { verbose = verboseOpt.HasValue(), showSchema = showSchemaOpt.HasValue(), outputJson = jsonOpt.HasValue(), outputJsonQuote = jsonSingleQuoteOpt.HasValue() ? '\'' : '"', outputJsonIndent = jsonNoIndentOpt.HasValue() ? null : " ", interactive = interactiveOpt.HasValue(), namespaceFile = namespaceOpt.Value(), rows = rowsOpt.Values }; return config.OnExecuteAsync().Result; }); }); } /// /// Read a from the current stream position without moving the /// position. /// /// The stream to read from. /// The header read at the current position. /// /// If a header cannot be read from the current stream position for any /// reason. /// /// /// On success the stream's position is not changed. On error the stream position is /// undefined. /// private static async Task PeekHybridRowHeaderAsync(Stream stm) { try { Memory bytes = await PrintCommand.ReadFixedAsync(stm, HybridRowHeader.Size); return MemoryMarshal.Read(bytes.Span); } finally { stm.Seek(-HybridRowHeader.Size, SeekOrigin.Current); } } /// Reads a fixed length segment from a stream. /// The stream to read from. /// The length to read in bytes. /// A sequence of bytes read from the stream exactly long. /// /// if bytes cannot be read from the current /// stream position. /// private static async Task> ReadFixedAsync(Stream stm, int length) { Memory bytes = new byte[length].AsMemory(); Memory active = bytes; int bytesRead; do { bytesRead = await stm.ReadAsync(active); active = active.Slice(bytesRead); } while (bytesRead != 0); if (active.Length != 0) { throw new Exception("Failed to parse row header"); } return bytes; } private static void Refresh(ConsoleNative.ConsoleModes origMode, Layout layout, int length, long index, string str) { string[] lines = str.Split('\n'); int height = lines.Length + 2; int width = (from line in lines select line.Length).Max(); height = Math.Max(height, Console.WindowHeight); width = Math.Max(width, Console.WindowWidth); ConsoleNative.Mode = origMode & ~ConsoleNative.ConsoleModes.ENABLE_WRAP_AT_EOL_OUTPUT; Console.CursorVisible = false; ConsoleNative.SetBufferSize(width, height); Console.Clear(); if (layout != null) { Console.ForegroundColor = ConsoleColor.White; Console.WriteLine($"[{index}] Schema: {layout.SchemaId} {layout.Name}, Length: {length}"); } Console.ForegroundColor = ConsoleColor.Gray; Console.WriteLine(str); Console.SetWindowPosition(0, 0); } private static bool ShowInteractive(Layout layout, int length, long index, string str) { int origBufferHeight = Console.BufferHeight; int origBufferWidth = Console.BufferWidth; ConsoleNative.ConsoleModes origMode = ConsoleNative.Mode; try { PrintCommand.Refresh(origMode, layout, length, index, str); while (true) { ConsoleKeyInfo cki = Console.ReadKey(true); if ((cki.Modifiers & ConsoleModifiers.Alt) != 0 || (cki.Modifiers & ConsoleModifiers.Shift) != 0 || (cki.Modifiers & ConsoleModifiers.Control) != 0) { continue; } int top; int left; switch (cki.Key) { case ConsoleKey.Q: case ConsoleKey.Escape: return false; case ConsoleKey.Spacebar: case ConsoleKey.N: case ConsoleKey.Enter: return true; case ConsoleKey.R: PrintCommand.Refresh(origMode, layout, length, index, str); break; case ConsoleKey.Home: Console.SetWindowPosition(0, 0); break; case ConsoleKey.End: Console.SetWindowPosition(Console.WindowLeft, Console.BufferHeight - Console.WindowHeight); break; case ConsoleKey.PageDown: top = Console.WindowTop + Console.WindowHeight; top = Math.Min(top, Console.BufferHeight - Console.WindowHeight); Console.SetWindowPosition(Console.WindowLeft, top); break; case ConsoleKey.PageUp: top = Console.WindowTop - Console.WindowHeight; top = Math.Max(top, 0); Console.SetWindowPosition(Console.WindowLeft, top); break; case ConsoleKey.DownArrow: top = Console.WindowTop + 1; top = Math.Min(top, Console.BufferHeight - Console.WindowHeight); Console.SetWindowPosition(Console.WindowLeft, top); break; case ConsoleKey.UpArrow: top = Console.WindowTop - 1; top = Math.Max(top, 0); Console.SetWindowPosition(Console.WindowLeft, top); break; case ConsoleKey.RightArrow: left = Console.WindowLeft + 1; left = Math.Min(left, Console.BufferWidth - Console.WindowWidth); Console.SetWindowPosition(left, Console.WindowTop); break; case ConsoleKey.LeftArrow: left = Console.WindowLeft - 1; left = Math.Max(left, 0); Console.SetWindowPosition(left, Console.WindowTop); break; } } } finally { ConsoleNative.Mode = origMode; Console.ResetColor(); origBufferWidth = Math.Max(Console.WindowWidth, origBufferWidth); origBufferHeight = Math.Max(Console.WindowHeight, origBufferHeight); Console.SetBufferSize(origBufferWidth, origBufferHeight); Console.Clear(); Console.CursorVisible = true; } } /// Convert a text stream containing a sequence of HEX characters into a byte sequence. /// The stream to read the text from. /// The entire contents of the stream interpreted as HEX characters. /// Any character that is not a HEX character is ignored. private static async Task> HexTextStreamToBinaryAsync(Stream stm) { using (TextReader reader = new StreamReader(stm)) { string rowAsText = await reader.ReadToEndAsync(); string trimmed = string.Concat( from c in rowAsText where (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') select c); return ByteConverter.ToBytes(trimmed).AsMemory(); } } private async Task OnExecuteAsync() { LayoutResolver globalResolver = await SchemaUtil.CreateResolverAsync(this.namespaceFile, this.verbose); MemorySpanResizer resizer = new MemorySpanResizer(PrintCommand.InitialCapacity); foreach (string rowFile in this.rows) { using (Stream stm = new FileStream(rowFile, FileMode.Open)) { // Detect if it is a text or binary file via the encoding of the magic number at the // beginning of the HybridRow header. int magicNumber = stm.ReadByte(); stm.Seek(-1, SeekOrigin.Current); if (magicNumber == -1) { continue; // empty file } Result r = Result.Failure; if (magicNumber == (int)HybridRowVersion.V1) { HybridRowHeader header = await PrintCommand.PeekHybridRowHeaderAsync(stm); if (header.SchemaId == SystemSchema.SegmentSchemaId) { r = await this.PrintRecordIOAsync(stm, globalResolver, resizer); } else { Memory rowAsBinary = await PrintCommand.ReadFixedAsync(stm, (int)stm.Length); r = this.PrintOneRow(rowAsBinary, 0, globalResolver); } } else if (char.ToUpper((char)magicNumber, CultureInfo.InvariantCulture) == HybridRowVersion.V1.ToString("X")[0]) { // Convert hex text file to binary. Memory rowAsBinary = await PrintCommand.HexTextStreamToBinaryAsync(stm); r = this.PrintOneRow(rowAsBinary, 0, globalResolver); } if (r == Result.Canceled) { return 0; } if (r != Result.Success) { await Console.Error.WriteLineAsync($"Error reading row at {rowFile}"); return -1; } } } return 0; } /// Print all records from a HybridRow RecordIO file. /// Stream containing the HybridRow RecordIO content. /// The global resolver to use when segments don't contain embedded SDL. /// The resizer for allocating row buffers. /// Success if the print is successful, an error code otherwise. private async Task PrintRecordIOAsync(Stream stm, LayoutResolver globalResolver, MemorySpanResizer resizer) { LayoutResolver segmentResolver = globalResolver; // Read a RecordIO stream. long index = 0; Result r = await stm.ReadRecordIOAsync( record => this.PrintOneRow(record, index++, segmentResolver), segment => { r = SegmentSerializer.Read(segment.Span, globalResolver, out Segment s); if (r != Result.Success) { return r; } segmentResolver = string.IsNullOrWhiteSpace(s.SDL) ? globalResolver : SchemaUtil.LoadFromSdl(s.SDL, this.verbose, globalResolver); if (this.showSchema) { string str = string.IsNullOrWhiteSpace(s.SDL) ? "" : s.SDL; if (!this.interactive) { Console.WriteLine(str); } else { if (!PrintCommand.ShowInteractive(null, 0, -1, str)) { return Result.Canceled; } } } return Result.Success; }, resizer); return r; } /// Print a single HybridRow record. /// The raw bytes of the row. /// /// A 0-based index of the row relative to its outer container (for display /// purposes only). /// /// The resolver for nested contents within the row. /// Success if the print is successful, an error code otherwise. private Result PrintOneRow(Memory buffer, long index, LayoutResolver resolver) { RowBuffer row = new RowBuffer(buffer.Span, HybridRowVersion.V1, resolver); RowReader reader = new RowReader(ref row); Result r; string str; if (this.outputJson) { r = reader.ToJson(new RowReaderJsonSettings(this.outputJsonIndent, this.outputJsonQuote), out str); } else { r = DiagnosticConverter.ReaderToString(ref reader, out str); } if (r != Result.Success) { return r; } Layout layout = resolver.Resolve(row.Header.SchemaId); if (!this.interactive) { Console.WriteLine($"Schema: {layout.SchemaId} {layout.Name}, Length: {row.Length}"); Console.WriteLine(str); } else { if (!PrintCommand.ShowInteractive(layout, row.Length, index, str)) { return Result.Canceled; } } return Result.Success; } } }