From a3359b29416249dceb3a6a1871670fa600233148 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 15:45:06 +1000 Subject: [PATCH 1/6] Work-in-progress pattern executor --- src/SeqCli/PlainText/BuiltInPatterns.cs | 24 +++++++++ src/SeqCli/PlainText/Parsers/IdentifierEx.cs | 14 +++++ src/SeqCli/PlainText/Parsers/SpanEx.cs | 24 +++++++++ src/SeqCli/PlainText/Pattern.cs | 53 ++++++++++++++++++ src/SeqCli/PlainText/PatternBuilder.cs | 9 ++++ src/SeqCli/PlainText/PatternElement.cs | 20 +++++++ src/SeqCli/PlainText/ReifiedProperties.cs | 7 +++ .../PlainText/FrameReaderTests.cs | 25 ++------- test/SeqCli.Tests/PlainText/PatternTests.cs | 54 +++++++++++++++++++ 9 files changed, 210 insertions(+), 20 deletions(-) create mode 100644 src/SeqCli/PlainText/BuiltInPatterns.cs create mode 100644 src/SeqCli/PlainText/Parsers/IdentifierEx.cs create mode 100644 src/SeqCli/PlainText/Parsers/SpanEx.cs create mode 100644 src/SeqCli/PlainText/Pattern.cs create mode 100644 src/SeqCli/PlainText/PatternBuilder.cs create mode 100644 src/SeqCli/PlainText/PatternElement.cs create mode 100644 src/SeqCli/PlainText/ReifiedProperties.cs create mode 100644 test/SeqCli.Tests/PlainText/PatternTests.cs diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs new file mode 100644 index 00000000..bed88b6d --- /dev/null +++ b/src/SeqCli/PlainText/BuiltInPatterns.cs @@ -0,0 +1,24 @@ +using SeqCli.PlainText.Parsers; +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText +{ + static class BuiltInPatterns + { + public static TextParser Identifier { get; } = + IdentifierEx.CStyle + .Select(span => (object) span); + + public static TextParser MultiLineMessage { get; } = + SpanEx.MatchedBy( + Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") + .IgnoreThen(Character.AnyChar.Many())) + .Select(span => (object)span); + + public static TextParser LiteralText(string literalText) + { + return Span.EqualTo(literalText).Select(span => (object) span); + } + } +} diff --git a/src/SeqCli/PlainText/Parsers/IdentifierEx.cs b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs new file mode 100644 index 00000000..89fe36c4 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs @@ -0,0 +1,14 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class IdentifierEx + { + public static TextParser CStyle { get; } = + SpanEx.MatchedBy( + Character.Letter.Or(Character.EqualTo('_')) + .IgnoreThen(Character.LetterOrDigit).Or(Character.EqualTo('_')).Many()); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Parsers/SpanEx.cs b/src/SeqCli/PlainText/Parsers/SpanEx.cs new file mode 100644 index 00000000..26e45087 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/SpanEx.cs @@ -0,0 +1,24 @@ +using Superpower; +using Superpower.Model; + +namespace SeqCli.PlainText.Parsers +{ + static class SpanEx + { + public static TextParser MatchedBy(TextParser parser) + { + return i => + { + var result = parser(i); + + if (!result.HasValue) + return Result.CastEmpty(result); + + return Result.Value( + i.Until(result.Remainder), + i, + result.Remainder); + }; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Pattern.cs b/src/SeqCli/PlainText/Pattern.cs new file mode 100644 index 00000000..8faeabc3 --- /dev/null +++ b/src/SeqCli/PlainText/Pattern.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText +{ + class Pattern + { + readonly PatternElement[] _elements; + + public Pattern(IEnumerable elements) + { + _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements)); + if (_elements.Length == 0) + throw new ArgumentException("A match pattern must contain at least one element."); + } + + public TextParser FrameStart => _elements[0].Parser; + + public (IDictionary, string) Match(string frame) + { + var input = new TextSpan(frame); + var result = new Dictionary(); + + var remainder = input; + foreach (var element in _elements) + { + var match = element.Parser(remainder); + if (!match.HasValue) + { + if (remainder.IsAtEnd || Span.WhiteSpace.IsMatch(remainder)) + return (result, null); + + return (result, remainder.ToStringValue()); + } + + remainder = match.Remainder; + + if (!element.IsIgnored) + { + if (match.Value != null || !element.IsOptional) + result.Add(element.Name, match.Value); + } + } + + return (result, null); + } + } +} diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs new file mode 100644 index 00000000..9985ea68 --- /dev/null +++ b/src/SeqCli/PlainText/PatternBuilder.cs @@ -0,0 +1,9 @@ +namespace SeqCli.PlainText +{ + static class PatternBuilder + { + public static Pattern DefaultPattern = new Pattern(new[] { + new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) + }); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternElement.cs b/src/SeqCli/PlainText/PatternElement.cs new file mode 100644 index 00000000..53d30eae --- /dev/null +++ b/src/SeqCli/PlainText/PatternElement.cs @@ -0,0 +1,20 @@ +using System; +using Superpower; + +namespace SeqCli.PlainText +{ + class PatternElement + { + public PatternElement(TextParser parser, string name = null, bool isOptional = false) + { + Parser = parser ?? throw new ArgumentNullException(nameof(parser)); + Name = name; + IsOptional = isOptional; + } + + public TextParser Parser { get; } + public string Name { get; } + public bool IsOptional { get; } + public bool IsIgnored => Name == null; + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/ReifiedProperties.cs b/src/SeqCli/PlainText/ReifiedProperties.cs new file mode 100644 index 00000000..afb3ee5b --- /dev/null +++ b/src/SeqCli/PlainText/ReifiedProperties.cs @@ -0,0 +1,7 @@ +namespace SeqCli.PlainText +{ + static class ReifiedProperties + { + public const string Message = "@m"; + } +} diff --git a/test/SeqCli.Tests/PlainText/FrameReaderTests.cs b/test/SeqCli.Tests/PlainText/FrameReaderTests.cs index c76ca258..d1f97058 100644 --- a/test/SeqCli.Tests/PlainText/FrameReaderTests.cs +++ b/test/SeqCli.Tests/PlainText/FrameReaderTests.cs @@ -4,6 +4,7 @@ using System.Text; using System.Threading.Tasks; using SeqCli.PlainText; +using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; using Superpower.Parsers; @@ -12,23 +13,7 @@ namespace SeqCli.Tests.PlainText { public class FrameReaderTests - { - static TextParser SpanMatchedBy(TextParser parser) - { - return i => - { - var result = parser(i); - - if (!result.HasValue) - return Result.CastEmpty(result); - - return Result.Value( - i.Until(result.Remainder), - i, - result.Remainder); - }; - } - + { [Fact] public async Task SplitsLinesIntoFrames() { @@ -38,7 +23,7 @@ public async Task SplitsLinesIntoFrames() var reader = new FrameReader( new StringReader(source.ToString()), - SpanMatchedBy(Character.Letter), + SpanEx.MatchedBy(Character.Letter), TimeSpan.FromMilliseconds(1)); var first = await reader.TryReadAsync(); @@ -58,7 +43,7 @@ public async Task TerminatesWhenNoLinesArePresent() { var reader = new FrameReader( new StringReader(""), - SpanMatchedBy(Character.Letter), + SpanEx.MatchedBy(Character.Letter), TimeSpan.FromMilliseconds(1)); var none = await reader.TryReadAsync(); @@ -76,7 +61,7 @@ public async Task CollectsTrailingLines() source.AppendLine("third"); source.AppendLine(" and yet more"); - var frames = await ReadAllFrames(source.ToString(), SpanMatchedBy(Character.Letter)); + var frames = await ReadAllFrames(source.ToString(), SpanEx.MatchedBy(Character.Letter)); Assert.Equal(3, frames.Length); Assert.StartsWith("first", frames[0].Value); Assert.EndsWith("and more" + Environment.NewLine, frames[0].Value); diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/PatternTests.cs new file mode 100644 index 00000000..6e1821a7 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/PatternTests.cs @@ -0,0 +1,54 @@ +using System; +using SeqCli.PlainText; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class PatternTests + { + [Fact] + public void TheDefaultPatternMatchesMultilineMessages() + { + var frame = $"Hello,{Environment.NewLine} world!"; + var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + Assert.Null(remainder); + Assert.Single(properties, p => p.Key == ReifiedProperties.Message && + ((TextSpan)p.Value).ToStringValue() == frame); + } + + [Fact] + public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() + { + var frame = " world"; + var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + Assert.Empty(properties); + Assert.Equal(frame, remainder); + } + + static Pattern ClassMethodPattern { get; } = new Pattern(new[] + { + new PatternElement(BuiltInPatterns.Identifier, "class"), + new PatternElement(BuiltInPatterns.LiteralText(".")), + new PatternElement(BuiltInPatterns.Identifier, "method") + }); + + [Fact] + public void PatternsExtractElements() + { + var pattern = ClassMethodPattern; + + var frame = "this.that"; + var (properties, remainder) = pattern.Match(frame); + Assert.Null(remainder); + Assert.Equal("this", properties["class"].ToString()); + Assert.Equal("that", properties["method"].ToString()); + } + + [Fact] + public void TheFirstPatternElementIsExposed() + { + Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); + } + } +} \ No newline at end of file From 4d3120c9816375b5c7e37bf32131bc655e075dc5 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 16:09:58 +1000 Subject: [PATCH 2/6] Non-greedy matching --- src/SeqCli/PlainText/BuiltInPatterns.cs | 33 ++++++++++++++ src/SeqCli/PlainText/Parsers/IdentifierEx.cs | 2 +- test/SeqCli.Tests/PlainText/PatternTests.cs | 48 ++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs index bed88b6d..02be19fa 100644 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ b/src/SeqCli/PlainText/BuiltInPatterns.cs @@ -1,5 +1,6 @@ using SeqCli.PlainText.Parsers; using Superpower; +using Superpower.Model; using Superpower.Parsers; namespace SeqCli.PlainText @@ -15,10 +16,42 @@ static class BuiltInPatterns Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") .IgnoreThen(Character.AnyChar.Many())) .Select(span => (object)span); + + public static TextParser MultiLineContent { get; } = + SpanEx.MatchedBy(Character.AnyChar.Many()) + .Select(span => (object)span); + + public static TextParser SingleLineContent { get; } = + SpanEx.MatchedBy(Character.ExceptIn('\r', '\n').Many()) + .Select(span => (object)span); public static TextParser LiteralText(string literalText) { return Span.EqualTo(literalText).Select(span => (object) span); } + + public static TextParser NonGreedyContent(PatternElement[] following) + { + if (following.Length == 0) + return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); + + var rest = following[0].Parser; + for (var i = 1; i < following.Length; ++i) + { + rest = rest.IgnoreThen(following[i].Parser); + } + + return i => + { + var remainder = i; + while (!rest.IsMatch(remainder)) + { + remainder = remainder.ConsumeChar().Remainder; + } + + var span = i.Until(remainder); + return Result.Value((object) span, i, remainder); + }; + } } } diff --git a/src/SeqCli/PlainText/Parsers/IdentifierEx.cs b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs index 89fe36c4..c7248029 100644 --- a/src/SeqCli/PlainText/Parsers/IdentifierEx.cs +++ b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs @@ -9,6 +9,6 @@ static class IdentifierEx public static TextParser CStyle { get; } = SpanEx.MatchedBy( Character.Letter.Or(Character.EqualTo('_')) - .IgnoreThen(Character.LetterOrDigit).Or(Character.EqualTo('_')).Many()); + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())); } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/PatternTests.cs index 6e1821a7..e899cc4b 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/PatternTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using SeqCli.PlainText; using Superpower.Model; using Xunit; @@ -50,5 +51,52 @@ public void TheFirstPatternElementIsExposed() { Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); } + + [Fact] + public void SingleLineContentMatchesUntilEol() + { + var pattern = new Pattern(new[] + { + new PatternElement(BuiltInPatterns.Identifier, "first"), + new PatternElement(BuiltInPatterns.LiteralText(" ")), + new PatternElement(BuiltInPatterns.SingleLineContent, "content"), + new PatternElement(BuiltInPatterns.LiteralText(" (")), + new PatternElement(BuiltInPatterns.Identifier, "last"), + new PatternElement(BuiltInPatterns.LiteralText(")")) + }); + + var frame = "abc def ghi (jkl)"; + var (properties, remainder) = pattern.Match(frame); + Assert.Null(remainder); + Assert.Equal("abc", properties["first"].ToString()); + Assert.Equal("def ghi (jkl)", properties["content"].ToString()); + } + + [Fact] + public void NonGreedyContentStopsMatchingWhenFollowingTokensMatch() + { + // It's likely we'll only be able to get one or two tokens into + // the "following" list, since they effectively become "mandatory" + var following = new[] + { + new PatternElement(BuiltInPatterns.LiteralText(" (")), + new PatternElement(BuiltInPatterns.Identifier, "last"), + new PatternElement(BuiltInPatterns.LiteralText(")")) + }; + + var pattern = new Pattern(new[] + { + new PatternElement(BuiltInPatterns.Identifier, "first"), + new PatternElement(BuiltInPatterns.LiteralText(" ")), + new PatternElement(BuiltInPatterns.NonGreedyContent(following), "content"), + }.Concat(following)); + + var frame = "abc def ghi (jkl)"; + var (properties, remainder) = pattern.Match(frame); + Assert.Null(remainder); + Assert.Equal("abc", properties["first"].ToString()); + Assert.Equal("def ghi", properties["content"].ToString()); + Assert.Equal("jkl", properties["last"].ToString()); + } } } \ No newline at end of file From 9f02d73d50dd2c3ea5403af12efd3a7718a6d47a Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 16:20:04 +1000 Subject: [PATCH 3/6] Some notes to self --- src/SeqCli/PlainText/BuiltInPatterns.cs | 3 +++ src/SeqCli/PlainText/Parsers/SpanEx.cs | 4 ++++ src/SeqCli/PlainText/PatternBuilder.cs | 14 +++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs index 02be19fa..01f649d7 100644 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ b/src/SeqCli/PlainText/BuiltInPatterns.cs @@ -11,6 +11,9 @@ static class BuiltInPatterns IdentifierEx.CStyle .Select(span => (object) span); + public static TextParser Token { get; } = + SpanEx.NonWhiteSpace.Select(span => (object)span); + public static TextParser MultiLineMessage { get; } = SpanEx.MatchedBy( Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") diff --git a/src/SeqCli/PlainText/Parsers/SpanEx.cs b/src/SeqCli/PlainText/Parsers/SpanEx.cs index 26e45087..0564a735 100644 --- a/src/SeqCli/PlainText/Parsers/SpanEx.cs +++ b/src/SeqCli/PlainText/Parsers/SpanEx.cs @@ -1,5 +1,6 @@ using Superpower; using Superpower.Model; +using Superpower.Parsers; namespace SeqCli.PlainText.Parsers { @@ -20,5 +21,8 @@ public static TextParser MatchedBy(TextParser parser) result.Remainder); }; } + + public static TextParser NonWhiteSpace { get; } = + Span.WithoutAny(char.IsWhiteSpace); } } \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs index 9985ea68..7c3c3a61 100644 --- a/src/SeqCli/PlainText/PatternBuilder.cs +++ b/src/SeqCli/PlainText/PatternBuilder.cs @@ -2,8 +2,20 @@ { static class PatternBuilder { - public static Pattern DefaultPattern = new Pattern(new[] { + public static Pattern DefaultPattern { get; } = new Pattern(new[] { new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) }); + + // What we need to do here is: + // - for each parsed token + // - if it's literal text, map it an anonymous PatternElement with + // BuiltInPatterns.LiteralText() + // - otherwise, if it specifies no format, it's a named element with + // the BuiltInPatterns.Token parser + // - if it does specify a format, look up the parser based on the name, except + // - if the format is `$` it is BuiltInPatterns.SingleLineContent + // - if the format is `$$` it is BuiltInPatterns.MultiLineContent + // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the + // parser that follows it } } \ No newline at end of file From a5dfae984f8e1a58d55f4811ad8d398f2954e735 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 22 Feb 2018 12:12:28 +1000 Subject: [PATCH 4/6] Ensure API key is passed via the command --- src/SeqCli/Cli/Commands/LogCommand.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/Cli/Commands/LogCommand.cs b/src/SeqCli/Cli/Commands/LogCommand.cs index a4e091b2..c3d34fb2 100644 --- a/src/SeqCli/Cli/Commands/LogCommand.cs +++ b/src/SeqCli/Cli/Commands/LogCommand.cs @@ -102,7 +102,13 @@ protected override async Task Run() } var connection = _connectionFactory.Connect(_connection); - var result = await connection.Client.HttpClient.PostAsync(ApiConstants.IngestionEndpoint, content); + + var request = new HttpRequestMessage(HttpMethod.Post, ApiConstants.IngestionEndpoint) {Content = content}; + + if (_connection.IsApiKeySpecified) + request.Headers.Add("X-Seq-ApiKey", _connection.ApiKey); + + var result = await connection.Client.HttpClient.SendAsync(request); if (result.IsSuccessStatusCode) return 0; From a600cca08b7b92f3275fd43219dc78e47aa0ef3a Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 22 Feb 2018 20:30:30 +1000 Subject: [PATCH 5/6] Hook the frame reader and pattern matcher up to IngestCommand --- src/SeqCli/Cli/Commands/IngestCommand.cs | 30 +++- src/SeqCli/Ingestion/ClefLogEventReader.cs | 45 ++++++ src/SeqCli/Ingestion/ILogEventReader.cs | 10 ++ src/SeqCli/Ingestion/LogShipper.cs | 22 ++- src/SeqCli/PlainText/FrameReader.cs | 7 +- src/SeqCli/PlainText/LogEventBuilder.cs | 150 ++++++++++++++++++ .../PlainText/PlainTextLogEventReader.cs | 41 +++++ src/SeqCli/PlainText/ReifiedProperties.cs | 20 ++- src/SeqCli/PlainText/TextOnlyException.cs | 33 ++++ 9 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 src/SeqCli/Ingestion/ClefLogEventReader.cs create mode 100644 src/SeqCli/Ingestion/ILogEventReader.cs create mode 100644 src/SeqCli/PlainText/LogEventBuilder.cs create mode 100644 src/SeqCli/PlainText/PlainTextLogEventReader.cs create mode 100644 src/SeqCli/PlainText/TextOnlyException.cs diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index c69162e3..f15c41d9 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -19,6 +19,7 @@ using SeqCli.Cli.Features; using SeqCli.Connection; using SeqCli.Ingestion; +using SeqCli.PlainText; using Serilog; using Serilog.Core; using Serilog.Events; @@ -28,7 +29,7 @@ namespace SeqCli.Cli.Commands { [Command("ingest", "Send JSON log events from a file or `STDIN`", - Example = "seqcli ingest -i events.clef --filter=\"@Level <> 'Debug'\" -p Environment=Test")] + Example = "seqcli ingest -i events.clef --json --filter=\"@Level <> 'Debug'\" -p Environment=Test")] class IngestCommand : Command { readonly SeqConnectionFactory _connectionFactory; @@ -37,6 +38,7 @@ class IngestCommand : Command readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; string _filter; + bool _json; public IngestCommand(SeqConnectionFactory connectionFactory) { @@ -45,6 +47,10 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); + Options.Add("json", + "Read the events as JSON (the default assumes plain text)", + v => _json = true); + Options.Add("f=|filter=", "Filter expression to select a subset of events", v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); @@ -71,14 +77,22 @@ protected override async Task Run() ? new StreamReader(File.Open(_fileInputFeature.InputFilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) : null) - using (var reader = new LogEventReader(inputFile ?? Console.In)) { - return await LogShipper.ShipEvents( - _connectionFactory.Connect(_connection), - reader, - enrichers, - _invalidDataHandlingFeature.InvalidDataHandling, - filter); + var input = inputFile ?? Console.In; + + var reader = _json ? + (ILogEventReader)new ClefLogEventReader(input) : + new PlainTextLogEventReader(input); + + using (reader as IDisposable) + { + return await LogShipper.ShipEvents( + _connectionFactory.Connect(_connection), + reader, + enrichers, + _invalidDataHandlingFeature.InvalidDataHandling, + filter); + } } } catch (Exception ex) diff --git a/src/SeqCli/Ingestion/ClefLogEventReader.cs b/src/SeqCli/Ingestion/ClefLogEventReader.cs new file mode 100644 index 00000000..d29c351e --- /dev/null +++ b/src/SeqCli/Ingestion/ClefLogEventReader.cs @@ -0,0 +1,45 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Threading.Tasks; +using Serilog.Events; +using Serilog.Formatting.Compact.Reader; + +namespace SeqCli.Ingestion +{ + class ClefLogEventReader : ILogEventReader, IDisposable + { + readonly LogEventReader _reader; + + public ClefLogEventReader(TextReader input) + { + _reader = new LogEventReader(input ?? throw new ArgumentNullException(nameof(input))); + } + + public Task TryReadAsync() + { + if (_reader.TryRead(out var evt)) + return Task.FromResult(evt); + + return Task.FromResult(null); + } + + public void Dispose() + { + _reader.Dispose(); + } + } +} diff --git a/src/SeqCli/Ingestion/ILogEventReader.cs b/src/SeqCli/Ingestion/ILogEventReader.cs new file mode 100644 index 00000000..a7fd95fa --- /dev/null +++ b/src/SeqCli/Ingestion/ILogEventReader.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Serilog.Events; + +namespace SeqCli.Ingestion +{ + interface ILogEventReader + { + Task TryReadAsync(); + } +} diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index 6c3f9a18..969d955c 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -25,7 +25,6 @@ using Serilog.Core; using Serilog.Events; using Serilog.Formatting.Compact; -using Serilog.Formatting.Compact.Reader; namespace SeqCli.Ingestion { @@ -38,7 +37,7 @@ static class LogShipper public static async Task ShipEvents( SeqConnection connection, - LogEventReader reader, + ILogEventReader reader, List enrichers, InvalidDataHandling invalidDataHandling, Func filter = null) @@ -47,7 +46,7 @@ public static async Task ShipEvents( if (reader == null) throw new ArgumentNullException(nameof(reader)); if (enrichers == null) throw new ArgumentNullException(nameof(enrichers)); - var batch = ReadBatch(reader, filter, BatchSize, invalidDataHandling); + var batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling); while (batch.Length > 0) { StringContent content; @@ -67,7 +66,7 @@ public static async Task ShipEvents( if (result.IsSuccessStatusCode) { - batch = ReadBatch(reader, filter, BatchSize, invalidDataHandling); + batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling); continue; } @@ -80,7 +79,7 @@ public static async Task ShipEvents( Log.Error("Failed with status code {StatusCode}: {ErrorMessage}", result.StatusCode, - (string)error.ErrorMessage); + (string)error.Error); } catch { @@ -95,16 +94,23 @@ public static async Task ShipEvents( return 0; } - static LogEvent[] ReadBatch(LogEventReader reader, Func filter, - int count, InvalidDataHandling invalidDataHandling) + static async Task ReadBatchAsync( + ILogEventReader reader, + Func filter, + int count, + InvalidDataHandling invalidDataHandling) { var batch = new List(); do { try { - while (batch.Count < count && reader.TryRead(out var evt)) + while (batch.Count < count) { + var evt = await reader.TryReadAsync(); + if (evt == null) + break; + if (filter == null || filter(evt)) { batch.Add(evt); diff --git a/src/SeqCli/PlainText/FrameReader.cs b/src/SeqCli/PlainText/FrameReader.cs index c4328804..143025cd 100644 --- a/src/SeqCli/PlainText/FrameReader.cs +++ b/src/SeqCli/PlainText/FrameReader.cs @@ -21,7 +21,7 @@ namespace SeqCli.PlainText { - class FrameReader + class FrameReader : IDisposable { readonly TextReader _source; readonly TimeSpan _trailingLineArrivalDeadline; @@ -122,5 +122,10 @@ bool IsFrameStart(string line) return result.HasValue && result.Value.Length > 0; } } + + public void Dispose() + { + _unawaitedNextLine?.Dispose(); + } } } diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEventBuilder.cs new file mode 100644 index 00000000..88cd040b --- /dev/null +++ b/src/SeqCli/PlainText/LogEventBuilder.cs @@ -0,0 +1,150 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Serilog.Events; +using Serilog.Parsing; +using Superpower.Model; + +namespace SeqCli.PlainText +{ + static class LogEventBuilder + { + public static LogEvent FromProperties(IDictionary properties, string remainder) + { + var timestamp = GetTimestamp(properties); + var level = GetLevel(properties); + var exception = TryGetException(properties); + var messageTemplate = GetMessageTemplate(properties); + var props = GetLogEventProperties(properties, remainder); + + return new LogEvent( + timestamp, + level, + exception, + messageTemplate, + props); + } + + static readonly MessageTemplate NoMessage = new MessageTemplateParser().Parse(""); + + static MessageTemplate GetMessageTemplate(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Message, out var m) && + m is TextSpan ts) + { + var text = ts.ToStringValue(); + return new MessageTemplate(new MessageTemplateToken[] {new TextToken(text) }); + } + + return NoMessage; + } + + static LogEventLevel GetLevel(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Level, out var l) && + l is TextSpan ts && + LevelsByName.TryGetValue(ts.ToStringValue(), out var level)) + return level; + return LogEventLevel.Information; + } + + static Exception TryGetException(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Exception, out var x) && + x is TextSpan ts) + return new TextOnlyException(ts.ToStringValue()); + return null; + } + + static IEnumerable GetLogEventProperties(IDictionary properties, string remainder) + { + var payload = properties + .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key)) + .Select(p => new LogEventProperty(p.Key, new ScalarValue(p.Value))); + + if (remainder != null) + payload = payload.Concat(new[] + { + new LogEventProperty("@unmatched", new ScalarValue(remainder)) + }); + return payload; + } + + static DateTimeOffset GetTimestamp(IDictionary properties) + { + var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && + t is TextSpan span && + DateTimeOffset.TryParseExact(span.ToStringValue(), "o", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var ts) + ? ts + : DateTimeOffset.Now; + return timestamp; + } + + static readonly Dictionary LevelsByName = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["t"] = LogEventLevel.Verbose, + ["tr"] = LogEventLevel.Verbose, + ["trc"] = LogEventLevel.Verbose, + ["trce"] = LogEventLevel.Verbose, + ["trace"] = LogEventLevel.Verbose, + ["v"] = LogEventLevel.Verbose, + ["ver"] = LogEventLevel.Verbose, + ["vrb"] = LogEventLevel.Verbose, + ["verb"] = LogEventLevel.Verbose, + ["verbose"] = LogEventLevel.Verbose, + ["d"] = LogEventLevel.Debug, + ["de"] = LogEventLevel.Debug, + ["dbg"] = LogEventLevel.Debug, + ["deb"] = LogEventLevel.Debug, + ["dbug"] = LogEventLevel.Debug, + ["debu"] = LogEventLevel.Debug, + ["debub"] = LogEventLevel.Debug, + ["i"] = LogEventLevel.Information, + ["in"] = LogEventLevel.Information, + ["inf"] = LogEventLevel.Information, + ["info"] = LogEventLevel.Information, + ["information"] = LogEventLevel.Information, + ["w"] = LogEventLevel.Warning, + ["wa"] = LogEventLevel.Warning, + ["war"] = LogEventLevel.Warning, + ["wrn"] = LogEventLevel.Warning, + ["warn"] = LogEventLevel.Warning, + ["warning"] = LogEventLevel.Warning, + ["e"] = LogEventLevel.Error, + ["er"] = LogEventLevel.Error, + ["err"] = LogEventLevel.Error, + ["erro"] = LogEventLevel.Error, + ["eror"] = LogEventLevel.Error, + ["error"] = LogEventLevel.Error, + ["f"] = LogEventLevel.Fatal, + ["fa"] = LogEventLevel.Fatal, + ["ftl"] = LogEventLevel.Fatal, + ["fat"] = LogEventLevel.Fatal, + ["fatl"] = LogEventLevel.Fatal, + ["fatal"] = LogEventLevel.Fatal, + ["c"] = LogEventLevel.Fatal, + ["cr"] = LogEventLevel.Fatal, + ["crt"] = LogEventLevel.Fatal, + ["cri"] = LogEventLevel.Fatal, + ["crit"] = LogEventLevel.Fatal, + ["critical"] = LogEventLevel.Fatal + }; + + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs new file mode 100644 index 00000000..8f413c4b --- /dev/null +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using SeqCli.Ingestion; +using SeqCli.PlainText.Parsers; +using Serilog.Events; + +namespace SeqCli.PlainText +{ + class PlainTextLogEventReader : ILogEventReader, IDisposable + { + static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); + + readonly Pattern _pattern; + readonly FrameReader _reader; + + public PlainTextLogEventReader(TextReader input) + { + _pattern = PatternBuilder.DefaultPattern; + _reader = new FrameReader(input, SpanEx.MatchedBy(_pattern.FrameStart), TrailingLineArrivalDeadline); + } + + public async Task TryReadAsync() + { + var frame = await _reader.TryReadAsync(); + if (!frame.HasValue) + return null; + + if (frame.IsOrphan) + throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); + + var (properties, remainder) = _pattern.Match(frame.Value); + return LogEventBuilder.FromProperties(properties, remainder); + } + + public void Dispose() + { + _reader.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/ReifiedProperties.cs b/src/SeqCli/PlainText/ReifiedProperties.cs index afb3ee5b..aa9950be 100644 --- a/src/SeqCli/PlainText/ReifiedProperties.cs +++ b/src/SeqCli/PlainText/ReifiedProperties.cs @@ -1,7 +1,23 @@ -namespace SeqCli.PlainText +using System.Collections.Generic; + +namespace SeqCli.PlainText { static class ReifiedProperties { - public const string Message = "@m"; + public const string + Message = "@m", + Timestamp = "@t", + Level = "@l", + Exception = "@x"; + + static readonly HashSet All = new HashSet + { + Message, Timestamp, Level, Exception + }; + + public static bool IsReifiedProperty(string name) + { + return All.Contains(name); + } } } diff --git a/src/SeqCli/PlainText/TextOnlyException.cs b/src/SeqCli/PlainText/TextOnlyException.cs new file mode 100644 index 00000000..d0f67b60 --- /dev/null +++ b/src/SeqCli/PlainText/TextOnlyException.cs @@ -0,0 +1,33 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.PlainText +{ + class TextOnlyException : Exception + { + readonly string _toStringValue; + + public TextOnlyException(string toStringValue) + { + _toStringValue = toStringValue ?? throw new ArgumentNullException(nameof(toStringValue)); + } + + public override string ToString() + { + return _toStringValue; + } + } +} \ No newline at end of file From e639dec8d563bb508dd72821b974d27dbd9231ff Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 22 Feb 2018 21:04:20 +1000 Subject: [PATCH 6/6] LogEventBuilder tests --- src/SeqCli/PlainText/LogEventBuilder.cs | 4 +- .../PlainText/LogEventBuilderTests.cs | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEventBuilder.cs index 88cd040b..f5b252c2 100644 --- a/src/SeqCli/PlainText/LogEventBuilder.cs +++ b/src/SeqCli/PlainText/LogEventBuilder.cs @@ -89,8 +89,8 @@ static DateTimeOffset GetTimestamp(IDictionary properties) { var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && t is TextSpan span && - DateTimeOffset.TryParseExact(span.ToStringValue(), "o", CultureInfo.InvariantCulture, - DateTimeStyles.None, out var ts) + DateTimeOffset.TryParse(span.ToStringValue(), CultureInfo.InvariantCulture, + DateTimeStyles.AssumeLocal, out var ts) ? ts : DateTimeOffset.Now; return timestamp; diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs new file mode 100644 index 00000000..50a67383 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using SeqCli.PlainText; +using Serilog.Events; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class LogEventBuilderTests + { + [Fact] + public void SuppliedValuesAreUsed() + { + var properties = new Dictionary + { + ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), + ["@l"] = new TextSpan("WRN"), + ["@m"] = new TextSpan("Hello, world"), + ["@x"] = new TextSpan("EverythingFailedException"), + ["MachineName"] = new TextSpan("TP"), + ["Count"] = 42 + }; + + var remainder = "rem"; + var evt = LogEventBuilder.FromProperties(properties, remainder); + + Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o")); + Assert.Equal("Hello, world", evt.RenderMessage()); + Assert.Equal(LogEventLevel.Warning, evt.Level); + Assert.Equal("EverythingFailedException", evt.Exception.ToString()); + Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value); + Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value.ToString()); + Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value.ToString()); + } + + [Fact] + public void MissingValuesAreDefaulted() + { + var evt = LogEventBuilder.FromProperties(new Dictionary(), null); + + Assert.True(evt.Timestamp > DateTimeOffset.Now.AddSeconds(-5)); + Assert.Equal("", evt.RenderMessage()); + Assert.Equal(LogEventLevel.Information, evt.Level); + Assert.Null(evt.Exception); + Assert.Empty(evt.Properties); + } + } +} \ No newline at end of file