From 155b232a8634ebc5640b1e07640c39479c2143e2 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 13:26:05 +1000 Subject: [PATCH 01/26] Plain text ingestion WIP - read frames of input with trailing lines --- src/SeqCli/Csv/CsvTokenizer.cs | 6 +- src/SeqCli/PlainText/Frame.cs | 23 ++++ src/SeqCli/PlainText/FrameReader.cs | 126 ++++++++++++++++++ src/SeqCli/SeqCli.csproj | 2 +- .../PlainText/FrameReaderTests.cs | 104 +++++++++++++++ 5 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 src/SeqCli/PlainText/Frame.cs create mode 100644 src/SeqCli/PlainText/FrameReader.cs create mode 100644 test/SeqCli.Tests/PlainText/FrameReaderTests.cs diff --git a/src/SeqCli/Csv/CsvTokenizer.cs b/src/SeqCli/Csv/CsvTokenizer.cs index e7ec7411..656d1b74 100644 --- a/src/SeqCli/Csv/CsvTokenizer.cs +++ b/src/SeqCli/Csv/CsvTokenizer.cs @@ -8,7 +8,7 @@ namespace SeqCli.Csv { class CsvTokenizer : Tokenizer { - static readonly TextParser Content = Span.While(ch => ch != '"'); + static readonly TextParser Content = Span.WithoutAny(ch => ch == '"'); protected override IEnumerable> Tokenize(TextSpan span) { @@ -26,9 +26,9 @@ protected override IEnumerable> Tokenize(TextSpan span) if (!next.HasValue) yield break; var text = Content(next.Location); - while (text.HasValue) + while (text.HasValue || !text.Remainder.IsAtEnd) { - if (text.Value.Length > 0) + if (text.HasValue) { if (TryMatchSpecialContent(text.Value, out var specialTokenType) && !IsEscapedDoubleQuote(text.Remainder)) diff --git a/src/SeqCli/PlainText/Frame.cs b/src/SeqCli/PlainText/Frame.cs new file mode 100644 index 00000000..8458dbc0 --- /dev/null +++ b/src/SeqCli/PlainText/Frame.cs @@ -0,0 +1,23 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace SeqCli.PlainText +{ + struct Frame + { + public bool HasValue { get; set; } + public bool IsOrphan { get; set; } + public string Value { get; set; } + } +} diff --git a/src/SeqCli/PlainText/FrameReader.cs b/src/SeqCli/PlainText/FrameReader.cs new file mode 100644 index 00000000..c4328804 --- /dev/null +++ b/src/SeqCli/PlainText/FrameReader.cs @@ -0,0 +1,126 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Superpower; +using Superpower.Model; + +namespace SeqCli.PlainText +{ + class FrameReader + { + readonly TextReader _source; + readonly TimeSpan _trailingLineArrivalDeadline; + readonly TextParser _frameStart; + + string _unconsumedFirstLine; + Task _unawaitedNextLine; + + public FrameReader(TextReader source, TextParser frameStart, TimeSpan trailingLineArrivalDeadline) + { + _source = source ?? throw new ArgumentNullException(nameof(source)); + _frameStart = frameStart ?? throw new ArgumentNullException(nameof(frameStart)); + _trailingLineArrivalDeadline = trailingLineArrivalDeadline; + } + + public async Task TryReadAsync() + { + var valueBuilder = new StringBuilder(); + var hasValue = false; + + if (_unconsumedFirstLine != null) + { + valueBuilder.AppendLine(_unconsumedFirstLine); + _unconsumedFirstLine = null; + hasValue = true; + } + else if (_unawaitedNextLine != null) + { + var line = await _unawaitedNextLine; + _unawaitedNextLine = null; + if (line == null) + return new Frame(); + + valueBuilder.AppendLine(line); + hasValue = true; + + if (!IsFrameStart(line)) + return new Frame {HasValue = true, IsOrphan = true, Value = valueBuilder.ToString()}; + } + + Task readLine = null; + while (true) + { + readLine = readLine ?? Task.Run(_source.ReadLineAsync); + var index = Task.WaitAny(new Task[] {readLine}, _trailingLineArrivalDeadline); + if (index == -1) // Timeout + { + if (hasValue) + { + _unawaitedNextLine = readLine; + return new Frame {HasValue = true, Value = valueBuilder.ToString()}; + } + + // else, around we go! + } + else + { + var line = await readLine; + readLine = null; + if (line == null) + { + if (hasValue) + { + return new Frame {HasValue = true, Value = valueBuilder.ToString()}; + } + + return new Frame(); + } + + if (IsFrameStart(line)) + { + if (hasValue) + { + _unconsumedFirstLine = line; + return new Frame {HasValue = true, Value = valueBuilder.ToString()}; + } + + valueBuilder.AppendLine(line); + hasValue = true; + } + else + { + if (!hasValue) + { + valueBuilder.AppendLine(line); + return new Frame {HasValue = true, Value = valueBuilder.ToString(), IsOrphan = true}; + } + + valueBuilder.AppendLine(line); + } + } + } + + bool IsFrameStart(string line) + { + if (line == null) throw new ArgumentNullException(nameof(line)); + var result = _frameStart(new TextSpan(line)); + return result.HasValue && result.Value.Length > 0; + } + } + } +} diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index e0447852..0292c67e 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -25,7 +25,7 @@ - + diff --git a/test/SeqCli.Tests/PlainText/FrameReaderTests.cs b/test/SeqCli.Tests/PlainText/FrameReaderTests.cs new file mode 100644 index 00000000..c76ca258 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/FrameReaderTests.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using SeqCli.PlainText; +using Superpower; +using Superpower.Model; +using Superpower.Parsers; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class FrameReaderTests + { + static TextParser SpanMatchedBy(TextParser parser) + { + return i => + { + var result = parser(i); + + if (!result.HasValue) + return Result.CastEmpty(result); + + return Result.Value( + i.Until(result.Remainder), + i, + result.Remainder); + }; + } + + [Fact] + public async Task SplitsLinesIntoFrames() + { + var source = new StringBuilder(); + source.AppendLine("first"); + source.AppendLine("second"); + + var reader = new FrameReader( + new StringReader(source.ToString()), + SpanMatchedBy(Character.Letter), + TimeSpan.FromMilliseconds(1)); + + var first = await reader.TryReadAsync(); + Assert.True(first.HasValue); + Assert.Equal("first" + Environment.NewLine, first.Value); + + var second = await reader.TryReadAsync(); + Assert.True(second.HasValue); + Assert.Equal("second" + Environment.NewLine, second.Value); + + var empty = await reader.TryReadAsync(); + Assert.False(empty.HasValue); + } + + [Fact] + public async Task TerminatesWhenNoLinesArePresent() + { + var reader = new FrameReader( + new StringReader(""), + SpanMatchedBy(Character.Letter), + TimeSpan.FromMilliseconds(1)); + + var none = await reader.TryReadAsync(); + Assert.False(none.HasValue); + } + + [Fact] + public async Task CollectsTrailingLines() + { + var source = new StringBuilder(); + source.AppendLine("first"); + source.AppendLine(" some more"); + source.AppendLine(" and more"); + source.AppendLine("second"); + source.AppendLine("third"); + source.AppendLine(" and yet more"); + + var frames = await ReadAllFrames(source.ToString(), SpanMatchedBy(Character.Letter)); + Assert.Equal(3, frames.Length); + Assert.StartsWith("first", frames[0].Value); + Assert.EndsWith("and more" + Environment.NewLine, frames[0].Value); + } + + static async Task ReadAllFrames(string source, TextParser frameStart) + { + var reader = new FrameReader( + new StringReader(source), + frameStart, + TimeSpan.FromMilliseconds(1)); + + var result = new List(); + + var frame = await reader.TryReadAsync(); + while (frame.HasValue) + { + result.Add(frame); + frame = await reader.TryReadAsync(); + } + + return result.ToArray(); + } + } +} \ No newline at end of file From a3359b29416249dceb3a6a1871670fa600233148 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 15:45:06 +1000 Subject: [PATCH 02/26] Work-in-progress pattern executor --- src/SeqCli/PlainText/BuiltInPatterns.cs | 24 +++++++++ src/SeqCli/PlainText/Parsers/IdentifierEx.cs | 14 +++++ src/SeqCli/PlainText/Parsers/SpanEx.cs | 24 +++++++++ src/SeqCli/PlainText/Pattern.cs | 53 ++++++++++++++++++ src/SeqCli/PlainText/PatternBuilder.cs | 9 ++++ src/SeqCli/PlainText/PatternElement.cs | 20 +++++++ src/SeqCli/PlainText/ReifiedProperties.cs | 7 +++ .../PlainText/FrameReaderTests.cs | 25 ++------- test/SeqCli.Tests/PlainText/PatternTests.cs | 54 +++++++++++++++++++ 9 files changed, 210 insertions(+), 20 deletions(-) create mode 100644 src/SeqCli/PlainText/BuiltInPatterns.cs create mode 100644 src/SeqCli/PlainText/Parsers/IdentifierEx.cs create mode 100644 src/SeqCli/PlainText/Parsers/SpanEx.cs create mode 100644 src/SeqCli/PlainText/Pattern.cs create mode 100644 src/SeqCli/PlainText/PatternBuilder.cs create mode 100644 src/SeqCli/PlainText/PatternElement.cs create mode 100644 src/SeqCli/PlainText/ReifiedProperties.cs create mode 100644 test/SeqCli.Tests/PlainText/PatternTests.cs diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs new file mode 100644 index 00000000..bed88b6d --- /dev/null +++ b/src/SeqCli/PlainText/BuiltInPatterns.cs @@ -0,0 +1,24 @@ +using SeqCli.PlainText.Parsers; +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText +{ + static class BuiltInPatterns + { + public static TextParser Identifier { get; } = + IdentifierEx.CStyle + .Select(span => (object) span); + + public static TextParser MultiLineMessage { get; } = + SpanEx.MatchedBy( + Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") + .IgnoreThen(Character.AnyChar.Many())) + .Select(span => (object)span); + + public static TextParser LiteralText(string literalText) + { + return Span.EqualTo(literalText).Select(span => (object) span); + } + } +} diff --git a/src/SeqCli/PlainText/Parsers/IdentifierEx.cs b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs new file mode 100644 index 00000000..89fe36c4 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs @@ -0,0 +1,14 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class IdentifierEx + { + public static TextParser CStyle { get; } = + SpanEx.MatchedBy( + Character.Letter.Or(Character.EqualTo('_')) + .IgnoreThen(Character.LetterOrDigit).Or(Character.EqualTo('_')).Many()); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Parsers/SpanEx.cs b/src/SeqCli/PlainText/Parsers/SpanEx.cs new file mode 100644 index 00000000..26e45087 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/SpanEx.cs @@ -0,0 +1,24 @@ +using Superpower; +using Superpower.Model; + +namespace SeqCli.PlainText.Parsers +{ + static class SpanEx + { + public static TextParser MatchedBy(TextParser parser) + { + return i => + { + var result = parser(i); + + if (!result.HasValue) + return Result.CastEmpty(result); + + return Result.Value( + i.Until(result.Remainder), + i, + result.Remainder); + }; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Pattern.cs b/src/SeqCli/PlainText/Pattern.cs new file mode 100644 index 00000000..8faeabc3 --- /dev/null +++ b/src/SeqCli/PlainText/Pattern.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText +{ + class Pattern + { + readonly PatternElement[] _elements; + + public Pattern(IEnumerable elements) + { + _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements)); + if (_elements.Length == 0) + throw new ArgumentException("A match pattern must contain at least one element."); + } + + public TextParser FrameStart => _elements[0].Parser; + + public (IDictionary, string) Match(string frame) + { + var input = new TextSpan(frame); + var result = new Dictionary(); + + var remainder = input; + foreach (var element in _elements) + { + var match = element.Parser(remainder); + if (!match.HasValue) + { + if (remainder.IsAtEnd || Span.WhiteSpace.IsMatch(remainder)) + return (result, null); + + return (result, remainder.ToStringValue()); + } + + remainder = match.Remainder; + + if (!element.IsIgnored) + { + if (match.Value != null || !element.IsOptional) + result.Add(element.Name, match.Value); + } + } + + return (result, null); + } + } +} diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs new file mode 100644 index 00000000..9985ea68 --- /dev/null +++ b/src/SeqCli/PlainText/PatternBuilder.cs @@ -0,0 +1,9 @@ +namespace SeqCli.PlainText +{ + static class PatternBuilder + { + public static Pattern DefaultPattern = new Pattern(new[] { + new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) + }); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternElement.cs b/src/SeqCli/PlainText/PatternElement.cs new file mode 100644 index 00000000..53d30eae --- /dev/null +++ b/src/SeqCli/PlainText/PatternElement.cs @@ -0,0 +1,20 @@ +using System; +using Superpower; + +namespace SeqCli.PlainText +{ + class PatternElement + { + public PatternElement(TextParser parser, string name = null, bool isOptional = false) + { + Parser = parser ?? throw new ArgumentNullException(nameof(parser)); + Name = name; + IsOptional = isOptional; + } + + public TextParser Parser { get; } + public string Name { get; } + public bool IsOptional { get; } + public bool IsIgnored => Name == null; + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/ReifiedProperties.cs b/src/SeqCli/PlainText/ReifiedProperties.cs new file mode 100644 index 00000000..afb3ee5b --- /dev/null +++ b/src/SeqCli/PlainText/ReifiedProperties.cs @@ -0,0 +1,7 @@ +namespace SeqCli.PlainText +{ + static class ReifiedProperties + { + public const string Message = "@m"; + } +} diff --git a/test/SeqCli.Tests/PlainText/FrameReaderTests.cs b/test/SeqCli.Tests/PlainText/FrameReaderTests.cs index c76ca258..d1f97058 100644 --- a/test/SeqCli.Tests/PlainText/FrameReaderTests.cs +++ b/test/SeqCli.Tests/PlainText/FrameReaderTests.cs @@ -4,6 +4,7 @@ using System.Text; using System.Threading.Tasks; using SeqCli.PlainText; +using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; using Superpower.Parsers; @@ -12,23 +13,7 @@ namespace SeqCli.Tests.PlainText { public class FrameReaderTests - { - static TextParser SpanMatchedBy(TextParser parser) - { - return i => - { - var result = parser(i); - - if (!result.HasValue) - return Result.CastEmpty(result); - - return Result.Value( - i.Until(result.Remainder), - i, - result.Remainder); - }; - } - + { [Fact] public async Task SplitsLinesIntoFrames() { @@ -38,7 +23,7 @@ public async Task SplitsLinesIntoFrames() var reader = new FrameReader( new StringReader(source.ToString()), - SpanMatchedBy(Character.Letter), + SpanEx.MatchedBy(Character.Letter), TimeSpan.FromMilliseconds(1)); var first = await reader.TryReadAsync(); @@ -58,7 +43,7 @@ public async Task TerminatesWhenNoLinesArePresent() { var reader = new FrameReader( new StringReader(""), - SpanMatchedBy(Character.Letter), + SpanEx.MatchedBy(Character.Letter), TimeSpan.FromMilliseconds(1)); var none = await reader.TryReadAsync(); @@ -76,7 +61,7 @@ public async Task CollectsTrailingLines() source.AppendLine("third"); source.AppendLine(" and yet more"); - var frames = await ReadAllFrames(source.ToString(), SpanMatchedBy(Character.Letter)); + var frames = await ReadAllFrames(source.ToString(), SpanEx.MatchedBy(Character.Letter)); Assert.Equal(3, frames.Length); Assert.StartsWith("first", frames[0].Value); Assert.EndsWith("and more" + Environment.NewLine, frames[0].Value); diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/PatternTests.cs new file mode 100644 index 00000000..6e1821a7 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/PatternTests.cs @@ -0,0 +1,54 @@ +using System; +using SeqCli.PlainText; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class PatternTests + { + [Fact] + public void TheDefaultPatternMatchesMultilineMessages() + { + var frame = $"Hello,{Environment.NewLine} world!"; + var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + Assert.Null(remainder); + Assert.Single(properties, p => p.Key == ReifiedProperties.Message && + ((TextSpan)p.Value).ToStringValue() == frame); + } + + [Fact] + public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() + { + var frame = " world"; + var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + Assert.Empty(properties); + Assert.Equal(frame, remainder); + } + + static Pattern ClassMethodPattern { get; } = new Pattern(new[] + { + new PatternElement(BuiltInPatterns.Identifier, "class"), + new PatternElement(BuiltInPatterns.LiteralText(".")), + new PatternElement(BuiltInPatterns.Identifier, "method") + }); + + [Fact] + public void PatternsExtractElements() + { + var pattern = ClassMethodPattern; + + var frame = "this.that"; + var (properties, remainder) = pattern.Match(frame); + Assert.Null(remainder); + Assert.Equal("this", properties["class"].ToString()); + Assert.Equal("that", properties["method"].ToString()); + } + + [Fact] + public void TheFirstPatternElementIsExposed() + { + Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); + } + } +} \ No newline at end of file From 4d3120c9816375b5c7e37bf32131bc655e075dc5 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 16:09:58 +1000 Subject: [PATCH 03/26] Non-greedy matching --- src/SeqCli/PlainText/BuiltInPatterns.cs | 33 ++++++++++++++ src/SeqCli/PlainText/Parsers/IdentifierEx.cs | 2 +- test/SeqCli.Tests/PlainText/PatternTests.cs | 48 ++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs index bed88b6d..02be19fa 100644 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ b/src/SeqCli/PlainText/BuiltInPatterns.cs @@ -1,5 +1,6 @@ using SeqCli.PlainText.Parsers; using Superpower; +using Superpower.Model; using Superpower.Parsers; namespace SeqCli.PlainText @@ -15,10 +16,42 @@ static class BuiltInPatterns Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") .IgnoreThen(Character.AnyChar.Many())) .Select(span => (object)span); + + public static TextParser MultiLineContent { get; } = + SpanEx.MatchedBy(Character.AnyChar.Many()) + .Select(span => (object)span); + + public static TextParser SingleLineContent { get; } = + SpanEx.MatchedBy(Character.ExceptIn('\r', '\n').Many()) + .Select(span => (object)span); public static TextParser LiteralText(string literalText) { return Span.EqualTo(literalText).Select(span => (object) span); } + + public static TextParser NonGreedyContent(PatternElement[] following) + { + if (following.Length == 0) + return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); + + var rest = following[0].Parser; + for (var i = 1; i < following.Length; ++i) + { + rest = rest.IgnoreThen(following[i].Parser); + } + + return i => + { + var remainder = i; + while (!rest.IsMatch(remainder)) + { + remainder = remainder.ConsumeChar().Remainder; + } + + var span = i.Until(remainder); + return Result.Value((object) span, i, remainder); + }; + } } } diff --git a/src/SeqCli/PlainText/Parsers/IdentifierEx.cs b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs index 89fe36c4..c7248029 100644 --- a/src/SeqCli/PlainText/Parsers/IdentifierEx.cs +++ b/src/SeqCli/PlainText/Parsers/IdentifierEx.cs @@ -9,6 +9,6 @@ static class IdentifierEx public static TextParser CStyle { get; } = SpanEx.MatchedBy( Character.Letter.Or(Character.EqualTo('_')) - .IgnoreThen(Character.LetterOrDigit).Or(Character.EqualTo('_')).Many()); + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())); } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/PatternTests.cs index 6e1821a7..e899cc4b 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/PatternTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using SeqCli.PlainText; using Superpower.Model; using Xunit; @@ -50,5 +51,52 @@ public void TheFirstPatternElementIsExposed() { Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); } + + [Fact] + public void SingleLineContentMatchesUntilEol() + { + var pattern = new Pattern(new[] + { + new PatternElement(BuiltInPatterns.Identifier, "first"), + new PatternElement(BuiltInPatterns.LiteralText(" ")), + new PatternElement(BuiltInPatterns.SingleLineContent, "content"), + new PatternElement(BuiltInPatterns.LiteralText(" (")), + new PatternElement(BuiltInPatterns.Identifier, "last"), + new PatternElement(BuiltInPatterns.LiteralText(")")) + }); + + var frame = "abc def ghi (jkl)"; + var (properties, remainder) = pattern.Match(frame); + Assert.Null(remainder); + Assert.Equal("abc", properties["first"].ToString()); + Assert.Equal("def ghi (jkl)", properties["content"].ToString()); + } + + [Fact] + public void NonGreedyContentStopsMatchingWhenFollowingTokensMatch() + { + // It's likely we'll only be able to get one or two tokens into + // the "following" list, since they effectively become "mandatory" + var following = new[] + { + new PatternElement(BuiltInPatterns.LiteralText(" (")), + new PatternElement(BuiltInPatterns.Identifier, "last"), + new PatternElement(BuiltInPatterns.LiteralText(")")) + }; + + var pattern = new Pattern(new[] + { + new PatternElement(BuiltInPatterns.Identifier, "first"), + new PatternElement(BuiltInPatterns.LiteralText(" ")), + new PatternElement(BuiltInPatterns.NonGreedyContent(following), "content"), + }.Concat(following)); + + var frame = "abc def ghi (jkl)"; + var (properties, remainder) = pattern.Match(frame); + Assert.Null(remainder); + Assert.Equal("abc", properties["first"].ToString()); + Assert.Equal("def ghi", properties["content"].ToString()); + Assert.Equal("jkl", properties["last"].ToString()); + } } } \ No newline at end of file From 9f02d73d50dd2c3ea5403af12efd3a7718a6d47a Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 21 Feb 2018 16:20:04 +1000 Subject: [PATCH 04/26] Some notes to self --- src/SeqCli/PlainText/BuiltInPatterns.cs | 3 +++ src/SeqCli/PlainText/Parsers/SpanEx.cs | 4 ++++ src/SeqCli/PlainText/PatternBuilder.cs | 14 +++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs index 02be19fa..01f649d7 100644 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ b/src/SeqCli/PlainText/BuiltInPatterns.cs @@ -11,6 +11,9 @@ static class BuiltInPatterns IdentifierEx.CStyle .Select(span => (object) span); + public static TextParser Token { get; } = + SpanEx.NonWhiteSpace.Select(span => (object)span); + public static TextParser MultiLineMessage { get; } = SpanEx.MatchedBy( Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") diff --git a/src/SeqCli/PlainText/Parsers/SpanEx.cs b/src/SeqCli/PlainText/Parsers/SpanEx.cs index 26e45087..0564a735 100644 --- a/src/SeqCli/PlainText/Parsers/SpanEx.cs +++ b/src/SeqCli/PlainText/Parsers/SpanEx.cs @@ -1,5 +1,6 @@ using Superpower; using Superpower.Model; +using Superpower.Parsers; namespace SeqCli.PlainText.Parsers { @@ -20,5 +21,8 @@ public static TextParser MatchedBy(TextParser parser) result.Remainder); }; } + + public static TextParser NonWhiteSpace { get; } = + Span.WithoutAny(char.IsWhiteSpace); } } \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs index 9985ea68..7c3c3a61 100644 --- a/src/SeqCli/PlainText/PatternBuilder.cs +++ b/src/SeqCli/PlainText/PatternBuilder.cs @@ -2,8 +2,20 @@ { static class PatternBuilder { - public static Pattern DefaultPattern = new Pattern(new[] { + public static Pattern DefaultPattern { get; } = new Pattern(new[] { new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) }); + + // What we need to do here is: + // - for each parsed token + // - if it's literal text, map it an anonymous PatternElement with + // BuiltInPatterns.LiteralText() + // - otherwise, if it specifies no format, it's a named element with + // the BuiltInPatterns.Token parser + // - if it does specify a format, look up the parser based on the name, except + // - if the format is `$` it is BuiltInPatterns.SingleLineContent + // - if the format is `$$` it is BuiltInPatterns.MultiLineContent + // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the + // parser that follows it } } \ No newline at end of file From a600cca08b7b92f3275fd43219dc78e47aa0ef3a Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 22 Feb 2018 20:30:30 +1000 Subject: [PATCH 05/26] Hook the frame reader and pattern matcher up to IngestCommand --- src/SeqCli/Cli/Commands/IngestCommand.cs | 30 +++- src/SeqCli/Ingestion/ClefLogEventReader.cs | 45 ++++++ src/SeqCli/Ingestion/ILogEventReader.cs | 10 ++ src/SeqCli/Ingestion/LogShipper.cs | 22 ++- src/SeqCli/PlainText/FrameReader.cs | 7 +- src/SeqCli/PlainText/LogEventBuilder.cs | 150 ++++++++++++++++++ .../PlainText/PlainTextLogEventReader.cs | 41 +++++ src/SeqCli/PlainText/ReifiedProperties.cs | 20 ++- src/SeqCli/PlainText/TextOnlyException.cs | 33 ++++ 9 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 src/SeqCli/Ingestion/ClefLogEventReader.cs create mode 100644 src/SeqCli/Ingestion/ILogEventReader.cs create mode 100644 src/SeqCli/PlainText/LogEventBuilder.cs create mode 100644 src/SeqCli/PlainText/PlainTextLogEventReader.cs create mode 100644 src/SeqCli/PlainText/TextOnlyException.cs diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index c69162e3..f15c41d9 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -19,6 +19,7 @@ using SeqCli.Cli.Features; using SeqCli.Connection; using SeqCli.Ingestion; +using SeqCli.PlainText; using Serilog; using Serilog.Core; using Serilog.Events; @@ -28,7 +29,7 @@ namespace SeqCli.Cli.Commands { [Command("ingest", "Send JSON log events from a file or `STDIN`", - Example = "seqcli ingest -i events.clef --filter=\"@Level <> 'Debug'\" -p Environment=Test")] + Example = "seqcli ingest -i events.clef --json --filter=\"@Level <> 'Debug'\" -p Environment=Test")] class IngestCommand : Command { readonly SeqConnectionFactory _connectionFactory; @@ -37,6 +38,7 @@ class IngestCommand : Command readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; string _filter; + bool _json; public IngestCommand(SeqConnectionFactory connectionFactory) { @@ -45,6 +47,10 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); + Options.Add("json", + "Read the events as JSON (the default assumes plain text)", + v => _json = true); + Options.Add("f=|filter=", "Filter expression to select a subset of events", v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); @@ -71,14 +77,22 @@ protected override async Task Run() ? new StreamReader(File.Open(_fileInputFeature.InputFilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) : null) - using (var reader = new LogEventReader(inputFile ?? Console.In)) { - return await LogShipper.ShipEvents( - _connectionFactory.Connect(_connection), - reader, - enrichers, - _invalidDataHandlingFeature.InvalidDataHandling, - filter); + var input = inputFile ?? Console.In; + + var reader = _json ? + (ILogEventReader)new ClefLogEventReader(input) : + new PlainTextLogEventReader(input); + + using (reader as IDisposable) + { + return await LogShipper.ShipEvents( + _connectionFactory.Connect(_connection), + reader, + enrichers, + _invalidDataHandlingFeature.InvalidDataHandling, + filter); + } } } catch (Exception ex) diff --git a/src/SeqCli/Ingestion/ClefLogEventReader.cs b/src/SeqCli/Ingestion/ClefLogEventReader.cs new file mode 100644 index 00000000..d29c351e --- /dev/null +++ b/src/SeqCli/Ingestion/ClefLogEventReader.cs @@ -0,0 +1,45 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Threading.Tasks; +using Serilog.Events; +using Serilog.Formatting.Compact.Reader; + +namespace SeqCli.Ingestion +{ + class ClefLogEventReader : ILogEventReader, IDisposable + { + readonly LogEventReader _reader; + + public ClefLogEventReader(TextReader input) + { + _reader = new LogEventReader(input ?? throw new ArgumentNullException(nameof(input))); + } + + public Task TryReadAsync() + { + if (_reader.TryRead(out var evt)) + return Task.FromResult(evt); + + return Task.FromResult(null); + } + + public void Dispose() + { + _reader.Dispose(); + } + } +} diff --git a/src/SeqCli/Ingestion/ILogEventReader.cs b/src/SeqCli/Ingestion/ILogEventReader.cs new file mode 100644 index 00000000..a7fd95fa --- /dev/null +++ b/src/SeqCli/Ingestion/ILogEventReader.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Serilog.Events; + +namespace SeqCli.Ingestion +{ + interface ILogEventReader + { + Task TryReadAsync(); + } +} diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index 6c3f9a18..969d955c 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -25,7 +25,6 @@ using Serilog.Core; using Serilog.Events; using Serilog.Formatting.Compact; -using Serilog.Formatting.Compact.Reader; namespace SeqCli.Ingestion { @@ -38,7 +37,7 @@ static class LogShipper public static async Task ShipEvents( SeqConnection connection, - LogEventReader reader, + ILogEventReader reader, List enrichers, InvalidDataHandling invalidDataHandling, Func filter = null) @@ -47,7 +46,7 @@ public static async Task ShipEvents( if (reader == null) throw new ArgumentNullException(nameof(reader)); if (enrichers == null) throw new ArgumentNullException(nameof(enrichers)); - var batch = ReadBatch(reader, filter, BatchSize, invalidDataHandling); + var batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling); while (batch.Length > 0) { StringContent content; @@ -67,7 +66,7 @@ public static async Task ShipEvents( if (result.IsSuccessStatusCode) { - batch = ReadBatch(reader, filter, BatchSize, invalidDataHandling); + batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling); continue; } @@ -80,7 +79,7 @@ public static async Task ShipEvents( Log.Error("Failed with status code {StatusCode}: {ErrorMessage}", result.StatusCode, - (string)error.ErrorMessage); + (string)error.Error); } catch { @@ -95,16 +94,23 @@ public static async Task ShipEvents( return 0; } - static LogEvent[] ReadBatch(LogEventReader reader, Func filter, - int count, InvalidDataHandling invalidDataHandling) + static async Task ReadBatchAsync( + ILogEventReader reader, + Func filter, + int count, + InvalidDataHandling invalidDataHandling) { var batch = new List(); do { try { - while (batch.Count < count && reader.TryRead(out var evt)) + while (batch.Count < count) { + var evt = await reader.TryReadAsync(); + if (evt == null) + break; + if (filter == null || filter(evt)) { batch.Add(evt); diff --git a/src/SeqCli/PlainText/FrameReader.cs b/src/SeqCli/PlainText/FrameReader.cs index c4328804..143025cd 100644 --- a/src/SeqCli/PlainText/FrameReader.cs +++ b/src/SeqCli/PlainText/FrameReader.cs @@ -21,7 +21,7 @@ namespace SeqCli.PlainText { - class FrameReader + class FrameReader : IDisposable { readonly TextReader _source; readonly TimeSpan _trailingLineArrivalDeadline; @@ -122,5 +122,10 @@ bool IsFrameStart(string line) return result.HasValue && result.Value.Length > 0; } } + + public void Dispose() + { + _unawaitedNextLine?.Dispose(); + } } } diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEventBuilder.cs new file mode 100644 index 00000000..88cd040b --- /dev/null +++ b/src/SeqCli/PlainText/LogEventBuilder.cs @@ -0,0 +1,150 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Serilog.Events; +using Serilog.Parsing; +using Superpower.Model; + +namespace SeqCli.PlainText +{ + static class LogEventBuilder + { + public static LogEvent FromProperties(IDictionary properties, string remainder) + { + var timestamp = GetTimestamp(properties); + var level = GetLevel(properties); + var exception = TryGetException(properties); + var messageTemplate = GetMessageTemplate(properties); + var props = GetLogEventProperties(properties, remainder); + + return new LogEvent( + timestamp, + level, + exception, + messageTemplate, + props); + } + + static readonly MessageTemplate NoMessage = new MessageTemplateParser().Parse(""); + + static MessageTemplate GetMessageTemplate(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Message, out var m) && + m is TextSpan ts) + { + var text = ts.ToStringValue(); + return new MessageTemplate(new MessageTemplateToken[] {new TextToken(text) }); + } + + return NoMessage; + } + + static LogEventLevel GetLevel(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Level, out var l) && + l is TextSpan ts && + LevelsByName.TryGetValue(ts.ToStringValue(), out var level)) + return level; + return LogEventLevel.Information; + } + + static Exception TryGetException(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Exception, out var x) && + x is TextSpan ts) + return new TextOnlyException(ts.ToStringValue()); + return null; + } + + static IEnumerable GetLogEventProperties(IDictionary properties, string remainder) + { + var payload = properties + .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key)) + .Select(p => new LogEventProperty(p.Key, new ScalarValue(p.Value))); + + if (remainder != null) + payload = payload.Concat(new[] + { + new LogEventProperty("@unmatched", new ScalarValue(remainder)) + }); + return payload; + } + + static DateTimeOffset GetTimestamp(IDictionary properties) + { + var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && + t is TextSpan span && + DateTimeOffset.TryParseExact(span.ToStringValue(), "o", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var ts) + ? ts + : DateTimeOffset.Now; + return timestamp; + } + + static readonly Dictionary LevelsByName = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["t"] = LogEventLevel.Verbose, + ["tr"] = LogEventLevel.Verbose, + ["trc"] = LogEventLevel.Verbose, + ["trce"] = LogEventLevel.Verbose, + ["trace"] = LogEventLevel.Verbose, + ["v"] = LogEventLevel.Verbose, + ["ver"] = LogEventLevel.Verbose, + ["vrb"] = LogEventLevel.Verbose, + ["verb"] = LogEventLevel.Verbose, + ["verbose"] = LogEventLevel.Verbose, + ["d"] = LogEventLevel.Debug, + ["de"] = LogEventLevel.Debug, + ["dbg"] = LogEventLevel.Debug, + ["deb"] = LogEventLevel.Debug, + ["dbug"] = LogEventLevel.Debug, + ["debu"] = LogEventLevel.Debug, + ["debub"] = LogEventLevel.Debug, + ["i"] = LogEventLevel.Information, + ["in"] = LogEventLevel.Information, + ["inf"] = LogEventLevel.Information, + ["info"] = LogEventLevel.Information, + ["information"] = LogEventLevel.Information, + ["w"] = LogEventLevel.Warning, + ["wa"] = LogEventLevel.Warning, + ["war"] = LogEventLevel.Warning, + ["wrn"] = LogEventLevel.Warning, + ["warn"] = LogEventLevel.Warning, + ["warning"] = LogEventLevel.Warning, + ["e"] = LogEventLevel.Error, + ["er"] = LogEventLevel.Error, + ["err"] = LogEventLevel.Error, + ["erro"] = LogEventLevel.Error, + ["eror"] = LogEventLevel.Error, + ["error"] = LogEventLevel.Error, + ["f"] = LogEventLevel.Fatal, + ["fa"] = LogEventLevel.Fatal, + ["ftl"] = LogEventLevel.Fatal, + ["fat"] = LogEventLevel.Fatal, + ["fatl"] = LogEventLevel.Fatal, + ["fatal"] = LogEventLevel.Fatal, + ["c"] = LogEventLevel.Fatal, + ["cr"] = LogEventLevel.Fatal, + ["crt"] = LogEventLevel.Fatal, + ["cri"] = LogEventLevel.Fatal, + ["crit"] = LogEventLevel.Fatal, + ["critical"] = LogEventLevel.Fatal + }; + + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs new file mode 100644 index 00000000..8f413c4b --- /dev/null +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using SeqCli.Ingestion; +using SeqCli.PlainText.Parsers; +using Serilog.Events; + +namespace SeqCli.PlainText +{ + class PlainTextLogEventReader : ILogEventReader, IDisposable + { + static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); + + readonly Pattern _pattern; + readonly FrameReader _reader; + + public PlainTextLogEventReader(TextReader input) + { + _pattern = PatternBuilder.DefaultPattern; + _reader = new FrameReader(input, SpanEx.MatchedBy(_pattern.FrameStart), TrailingLineArrivalDeadline); + } + + public async Task TryReadAsync() + { + var frame = await _reader.TryReadAsync(); + if (!frame.HasValue) + return null; + + if (frame.IsOrphan) + throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); + + var (properties, remainder) = _pattern.Match(frame.Value); + return LogEventBuilder.FromProperties(properties, remainder); + } + + public void Dispose() + { + _reader.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/ReifiedProperties.cs b/src/SeqCli/PlainText/ReifiedProperties.cs index afb3ee5b..aa9950be 100644 --- a/src/SeqCli/PlainText/ReifiedProperties.cs +++ b/src/SeqCli/PlainText/ReifiedProperties.cs @@ -1,7 +1,23 @@ -namespace SeqCli.PlainText +using System.Collections.Generic; + +namespace SeqCli.PlainText { static class ReifiedProperties { - public const string Message = "@m"; + public const string + Message = "@m", + Timestamp = "@t", + Level = "@l", + Exception = "@x"; + + static readonly HashSet All = new HashSet + { + Message, Timestamp, Level, Exception + }; + + public static bool IsReifiedProperty(string name) + { + return All.Contains(name); + } } } diff --git a/src/SeqCli/PlainText/TextOnlyException.cs b/src/SeqCli/PlainText/TextOnlyException.cs new file mode 100644 index 00000000..d0f67b60 --- /dev/null +++ b/src/SeqCli/PlainText/TextOnlyException.cs @@ -0,0 +1,33 @@ +// Copyright 2018 Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace SeqCli.PlainText +{ + class TextOnlyException : Exception + { + readonly string _toStringValue; + + public TextOnlyException(string toStringValue) + { + _toStringValue = toStringValue ?? throw new ArgumentNullException(nameof(toStringValue)); + } + + public override string ToString() + { + return _toStringValue; + } + } +} \ No newline at end of file From e639dec8d563bb508dd72821b974d27dbd9231ff Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 22 Feb 2018 21:04:20 +1000 Subject: [PATCH 06/26] LogEventBuilder tests --- src/SeqCli/PlainText/LogEventBuilder.cs | 4 +- .../PlainText/LogEventBuilderTests.cs | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEventBuilder.cs index 88cd040b..f5b252c2 100644 --- a/src/SeqCli/PlainText/LogEventBuilder.cs +++ b/src/SeqCli/PlainText/LogEventBuilder.cs @@ -89,8 +89,8 @@ static DateTimeOffset GetTimestamp(IDictionary properties) { var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && t is TextSpan span && - DateTimeOffset.TryParseExact(span.ToStringValue(), "o", CultureInfo.InvariantCulture, - DateTimeStyles.None, out var ts) + DateTimeOffset.TryParse(span.ToStringValue(), CultureInfo.InvariantCulture, + DateTimeStyles.AssumeLocal, out var ts) ? ts : DateTimeOffset.Now; return timestamp; diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs new file mode 100644 index 00000000..50a67383 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using SeqCli.PlainText; +using Serilog.Events; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class LogEventBuilderTests + { + [Fact] + public void SuppliedValuesAreUsed() + { + var properties = new Dictionary + { + ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), + ["@l"] = new TextSpan("WRN"), + ["@m"] = new TextSpan("Hello, world"), + ["@x"] = new TextSpan("EverythingFailedException"), + ["MachineName"] = new TextSpan("TP"), + ["Count"] = 42 + }; + + var remainder = "rem"; + var evt = LogEventBuilder.FromProperties(properties, remainder); + + Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o")); + Assert.Equal("Hello, world", evt.RenderMessage()); + Assert.Equal(LogEventLevel.Warning, evt.Level); + Assert.Equal("EverythingFailedException", evt.Exception.ToString()); + Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value); + Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value.ToString()); + Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value.ToString()); + } + + [Fact] + public void MissingValuesAreDefaulted() + { + var evt = LogEventBuilder.FromProperties(new Dictionary(), null); + + Assert.True(evt.Timestamp > DateTimeOffset.Now.AddSeconds(-5)); + Assert.Equal("", evt.RenderMessage()); + Assert.Equal(LogEventLevel.Information, evt.Level); + Assert.Null(evt.Exception); + Assert.Empty(evt.Properties); + } + } +} \ No newline at end of file From 3d179624786f69ff39901634b5d84eeb135b13e3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 23 Feb 2018 21:25:07 +1000 Subject: [PATCH 07/26] Initial extraction pattern support --- src/SeqCli/Cli/Commands/IngestCommand.cs | 10 ++- .../PlainText/Extraction/MatcherAttribute.cs | 15 +++++ .../Matchers.cs} | 58 ++++++++++++++--- .../NameValueExtractor.cs} | 18 +++--- .../PlainText/Extraction/PatternBuilder.cs | 62 +++++++++++++++++++ .../{ => Extraction}/PatternElement.cs | 6 +- src/SeqCli/PlainText/{ => Framing}/Frame.cs | 0 .../PlainText/{ => Framing}/FrameReader.cs | 0 .../{ => LogEvents}/LogEventBuilder.cs | 0 .../{ => LogEvents}/TextOnlyException.cs | 0 .../PlainText/Parsers/TextParserExtensions.cs | 13 ++++ src/SeqCli/PlainText/PatternBuilder.cs | 21 ------- .../Patterns/CapturePatternExpression.cs | 16 +++++ .../PlainText/Patterns/ExtractionPattern.cs | 17 +++++ .../Patterns/ExtractionPatternExpression.cs | 6 ++ .../Patterns/ExtractionPatternParser.cs | 53 ++++++++++++++++ .../Patterns/LiteralTextPatternExpression.cs | 14 +++++ .../PlainText/PlainTextLogEventReader.cs | 15 +++-- .../PlainText/ExtractionPatternParserTests.cs | 59 ++++++++++++++++++ test/SeqCli.Tests/PlainText/PatternTests.cs | 49 ++++++++------- 20 files changed, 355 insertions(+), 77 deletions(-) create mode 100644 src/SeqCli/PlainText/Extraction/MatcherAttribute.cs rename src/SeqCli/PlainText/{BuiltInPatterns.cs => Extraction/Matchers.cs} (50%) rename src/SeqCli/PlainText/{Pattern.cs => Extraction/NameValueExtractor.cs} (64%) create mode 100644 src/SeqCli/PlainText/Extraction/PatternBuilder.cs rename src/SeqCli/PlainText/{ => Extraction}/PatternElement.cs (73%) rename src/SeqCli/PlainText/{ => Framing}/Frame.cs (100%) rename src/SeqCli/PlainText/{ => Framing}/FrameReader.cs (100%) rename src/SeqCli/PlainText/{ => LogEvents}/LogEventBuilder.cs (100%) rename src/SeqCli/PlainText/{ => LogEvents}/TextOnlyException.cs (100%) create mode 100644 src/SeqCli/PlainText/Parsers/TextParserExtensions.cs delete mode 100644 src/SeqCli/PlainText/PatternBuilder.cs create mode 100644 src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/ExtractionPattern.cs create mode 100644 src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs create mode 100644 src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs create mode 100644 test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index f15c41d9..ed7817ae 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -37,7 +37,7 @@ class IngestCommand : Command readonly FileInputFeature _fileInputFeature; readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; - string _filter; + string _filter, _pattern; bool _json; public IngestCommand(SeqConnectionFactory connectionFactory) @@ -47,6 +47,10 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); + Options.Add("p=|pattern=", + "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)", + v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); + Options.Add("json", "Read the events as JSON (the default assumes plain text)", v => _json = true); @@ -54,7 +58,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) Options.Add("f=|filter=", "Filter expression to select a subset of events", v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); - + _connection = Enable(); } @@ -82,7 +86,7 @@ protected override async Task Run() var reader = _json ? (ILogEventReader)new ClefLogEventReader(input) : - new PlainTextLogEventReader(input); + new PlainTextLogEventReader(input, _pattern); using (reader as IDisposable) { diff --git a/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs b/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs new file mode 100644 index 00000000..903c00cd --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace SeqCli.PlainText.Extraction +{ + [AttributeUsage(AttributeTargets.Property)] + class MatcherAttribute : Attribute + { + public string Name { get; } + + public MatcherAttribute(string name) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs similarity index 50% rename from src/SeqCli/PlainText/BuiltInPatterns.cs rename to src/SeqCli/PlainText/Extraction/Matchers.cs index 01f649d7..9f5c9fcc 100644 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -1,39 +1,77 @@ -using SeqCli.PlainText.Parsers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; using Superpower.Parsers; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { - static class BuiltInPatterns + // ReSharper disable UnusedMember.Global + static class Matchers { + [Matcher("ident")] public static TextParser Identifier { get; } = IdentifierEx.CStyle .Select(span => (object) span); - + + [Matcher("nat")] + public static TextParser Natural { get; } = + Numerics.Natural + .Select(span => (object) span); + + [Matcher("int")] + public static TextParser Integer { get; } = + Numerics.Integer + .Select(span => (object) span); + + [Matcher("token")] public static TextParser Token { get; } = SpanEx.NonWhiteSpace.Select(span => (object)span); - + + // Unclear whether we need to name this public static TextParser MultiLineMessage { get; } = SpanEx.MatchedBy( Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") .IgnoreThen(Character.AnyChar.Many())) .Select(span => (object)span); - + + [Matcher("lines")] public static TextParser MultiLineContent { get; } = - SpanEx.MatchedBy(Character.AnyChar.Many()) + Span.WithAll(ch => true) .Select(span => (object)span); - + + [Matcher("line")] public static TextParser SingleLineContent { get; } = - SpanEx.MatchedBy(Character.ExceptIn('\r', '\n').Many()) + from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n') + from _ in NewLine.OptionalOrDefault() + select (object) content; + + [Matcher("n")] + public static TextParser NewLine { get; } = + Span.EqualTo("\r\n").Or(Span.EqualTo("\n")) .Select(span => (object)span); + static readonly Dictionary> ByType = new Dictionary>( + from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties + let attr = pi.GetCustomAttribute() + where attr != null + select KeyValuePair.Create(attr.Name, (TextParser) pi.GetValue(null))); + + public static TextParser GetByType(string type) + { + if (type == null) throw new ArgumentNullException(nameof(type)); + return ByType[type]; + } + public static TextParser LiteralText(string literalText) { return Span.EqualTo(literalText).Select(span => (object) span); } - public static TextParser NonGreedyContent(PatternElement[] following) + public static TextParser NonGreedyContent(params PatternElement[] following) { if (following.Length == 0) return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); diff --git a/src/SeqCli/PlainText/Pattern.cs b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs similarity index 64% rename from src/SeqCli/PlainText/Pattern.cs rename to src/SeqCli/PlainText/Extraction/NameValueExtractor.cs index 8faeabc3..243fe6a3 100644 --- a/src/SeqCli/PlainText/Pattern.cs +++ b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs @@ -1,29 +1,28 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; using Superpower; using Superpower.Model; using Superpower.Parsers; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { - class Pattern + class NameValueExtractor { readonly PatternElement[] _elements; - public Pattern(IEnumerable elements) + public NameValueExtractor(IEnumerable elements) { _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements)); if (_elements.Length == 0) - throw new ArgumentException("A match pattern must contain at least one element."); + throw new ArgumentException("An extraction pattern must contain at least one element."); } - public TextParser FrameStart => _elements[0].Parser; + public TextParser StartMarker => _elements[0].Parser; - public (IDictionary, string) Match(string frame) + public (IDictionary, string) ExtractValues(string plainText) { - var input = new TextSpan(frame); + var input = new TextSpan(plainText); var result = new Dictionary(); var remainder = input; @@ -42,8 +41,7 @@ public Pattern(IEnumerable elements) if (!element.IsIgnored) { - if (match.Value != null || !element.IsOptional) - result.Add(element.Name, match.Value); + result.Add(element.Name, match.Value); } } diff --git a/src/SeqCli/PlainText/Extraction/PatternBuilder.cs b/src/SeqCli/PlainText/Extraction/PatternBuilder.cs new file mode 100644 index 00000000..0ae2aae7 --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/PatternBuilder.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using SeqCli.PlainText.Patterns; + +namespace SeqCli.PlainText.Extraction +{ + static class PatternCompiler + { + public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[] + { + new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) + }); + + public static NameValueExtractor Compile(ExtractionPattern pattern) + { + if (pattern == null) throw new ArgumentNullException(nameof(pattern)); + + var patternElements = new PatternElement[pattern.Elements.Count]; + for (var i = 0; i < pattern.Elements.Count; ++i) + { + var element = pattern.Elements[i]; + switch (element) + { + case LiteralTextPatternExpression text: + patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); + break; + case CapturePatternExpression capture when capture.Type == "*": + if (i < pattern.Elements.Count - 1) + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(patternElements[i + 1]), + capture.Name); + else + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(), // <- same as MultiLineContent + capture.Name); + break; + case CapturePatternExpression capture: + patternElements[i] = new PatternElement( + capture.Type == null ? Matchers.Token : Matchers.GetByType(capture.Type), + capture.Name); + break; + default: + throw new InvalidOperationException($"Element `{element}` not recognized."); + } + } + + return new NameValueExtractor(patternElements); + } + + // What we need to do here is: + // - for each parsed token + // - if it's literal text, map it an anonymous PatternElement with + // BuiltInPatterns.LiteralText() + // - otherwise, if it specifies no format, it's a named element with + // the BuiltInPatterns.Token parser + // - if it does specify a format, look up the parser based on the name, except + // - if the format is `$` it is BuiltInPatterns.SingleLineContent + // - if the format is `$$` it is BuiltInPatterns.MultiLineContent + // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the + // parser that follows it + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternElement.cs b/src/SeqCli/PlainText/Extraction/PatternElement.cs similarity index 73% rename from src/SeqCli/PlainText/PatternElement.cs rename to src/SeqCli/PlainText/Extraction/PatternElement.cs index 53d30eae..29670dd9 100644 --- a/src/SeqCli/PlainText/PatternElement.cs +++ b/src/SeqCli/PlainText/Extraction/PatternElement.cs @@ -1,20 +1,18 @@ using System; using Superpower; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { class PatternElement { - public PatternElement(TextParser parser, string name = null, bool isOptional = false) + public PatternElement(TextParser parser, string name = null) { Parser = parser ?? throw new ArgumentNullException(nameof(parser)); Name = name; - IsOptional = isOptional; } public TextParser Parser { get; } public string Name { get; } - public bool IsOptional { get; } public bool IsIgnored => Name == null; } } \ No newline at end of file diff --git a/src/SeqCli/PlainText/Frame.cs b/src/SeqCli/PlainText/Framing/Frame.cs similarity index 100% rename from src/SeqCli/PlainText/Frame.cs rename to src/SeqCli/PlainText/Framing/Frame.cs diff --git a/src/SeqCli/PlainText/FrameReader.cs b/src/SeqCli/PlainText/Framing/FrameReader.cs similarity index 100% rename from src/SeqCli/PlainText/FrameReader.cs rename to src/SeqCli/PlainText/Framing/FrameReader.cs diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs similarity index 100% rename from src/SeqCli/PlainText/LogEventBuilder.cs rename to src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs diff --git a/src/SeqCli/PlainText/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs similarity index 100% rename from src/SeqCli/PlainText/TextOnlyException.cs rename to src/SeqCli/PlainText/LogEvents/TextOnlyException.cs diff --git a/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs b/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs new file mode 100644 index 00000000..52d4aacb --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs @@ -0,0 +1,13 @@ +using Superpower; + +namespace SeqCli.PlainText.Parsers +{ + public static class TextParserExtensions + { + public static TextParser Cast(this TextParser parser) + where T : U + { + return parser.Select(t => (U) t); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs deleted file mode 100644 index 7c3c3a61..00000000 --- a/src/SeqCli/PlainText/PatternBuilder.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace SeqCli.PlainText -{ - static class PatternBuilder - { - public static Pattern DefaultPattern { get; } = new Pattern(new[] { - new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) - }); - - // What we need to do here is: - // - for each parsed token - // - if it's literal text, map it an anonymous PatternElement with - // BuiltInPatterns.LiteralText() - // - otherwise, if it specifies no format, it's a named element with - // the BuiltInPatterns.Token parser - // - if it does specify a format, look up the parser based on the name, except - // - if the format is `$` it is BuiltInPatterns.SingleLineContent - // - if the format is `$$` it is BuiltInPatterns.MultiLineContent - // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the - // parser that follows it - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs new file mode 100644 index 00000000..def5648d --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs @@ -0,0 +1,16 @@ +using System; + +namespace SeqCli.PlainText.Patterns +{ + class CapturePatternExpression : ExtractionPatternExpression + { + public string Name { get; } + public string Type { get; } + + public CapturePatternExpression(string name, string type) + { + Name = name; + Type = type; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs b/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs new file mode 100644 index 00000000..4b810ed9 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SeqCli.PlainText.Patterns +{ + class ExtractionPattern + { + public IReadOnlyList Elements { get; } + + public ExtractionPattern(IEnumerable items) + { + if (items == null) throw new ArgumentNullException(nameof(items)); + Elements = items.ToArray(); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs new file mode 100644 index 00000000..fca4ec75 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs @@ -0,0 +1,6 @@ +namespace SeqCli.PlainText.Patterns +{ + abstract class ExtractionPatternExpression + { + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs new file mode 100644 index 00000000..1038967a --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -0,0 +1,53 @@ +using System; +using SeqCli.PlainText.Parsers; +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Patterns +{ + static class ExtractionPatternParser + { + static readonly TextParser LiteralText = + Span.EqualTo("{{").Value('{').Try() + .Or(Span.EqualTo("}}").Value('}').Try()) + .Or(Character.ExceptIn('{', '}')) + .AtLeastOnce() + .Select(ch => new LiteralTextPatternExpression(new string(ch))); + + static readonly TextParser CaptureName = + SpanEx.MatchedBy( + Character.Letter.Or(Character.In('@', '_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => s.ToStringValue()); + + static readonly TextParser CaptureType = + SpanEx.MatchedBy(Character.EqualTo('*')) + .Or(SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many()))) + .Select(s => s.ToStringValue()); + + static readonly TextParser Capture = + from _ in Character.EqualTo('{') + from name in CaptureName.OptionalOrDefault() + from type in Character.EqualTo(':') + .IgnoreThen(CaptureType) + .OptionalOrDefault() + where name != null || type != null + from __ in Character.EqualTo('}') + select new CapturePatternExpression(name, type); + + static readonly TextParser Element = + LiteralText.Cast() + .Or(Capture.Cast()); + + static readonly TextParser Pattern = + Element.AtLeastOnce().AtEnd().Select(e => new ExtractionPattern(e)); + + public static ExtractionPattern Parse(string extractionPattern) + { + if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); + if (extractionPattern == "") throw new ParseException("Zero-length extraction patterns are not allowed."); + return Pattern.Parse(extractionPattern); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs b/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs new file mode 100644 index 00000000..a8399e81 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs @@ -0,0 +1,14 @@ +using System; + +namespace SeqCli.PlainText.Patterns +{ + class LiteralTextPatternExpression : ExtractionPatternExpression + { + public string Text { get; } + + public LiteralTextPatternExpression(string text) + { + Text = text ?? throw new ArgumentNullException(nameof(text)); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs index 8f413c4b..e6e68f28 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -2,7 +2,9 @@ using System.IO; using System.Threading.Tasks; using SeqCli.Ingestion; +using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Parsers; +using SeqCli.PlainText.Patterns; using Serilog.Events; namespace SeqCli.PlainText @@ -11,13 +13,16 @@ class PlainTextLogEventReader : ILogEventReader, IDisposable { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - readonly Pattern _pattern; + readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input) + public PlainTextLogEventReader(TextReader input, string extractionPattern) { - _pattern = PatternBuilder.DefaultPattern; - _reader = new FrameReader(input, SpanEx.MatchedBy(_pattern.FrameStart), TrailingLineArrivalDeadline); + _nameValueExtractor = string.IsNullOrEmpty(extractionPattern) ? + PatternCompiler.MultilineMessageExtractor : + PatternCompiler.Compile(ExtractionPatternParser.Parse(extractionPattern)); + + _reader = new FrameReader(input, SpanEx.MatchedBy(_nameValueExtractor.StartMarker), TrailingLineArrivalDeadline); } public async Task TryReadAsync() @@ -29,7 +34,7 @@ public async Task TryReadAsync() if (frame.IsOrphan) throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - var (properties, remainder) = _pattern.Match(frame.Value); + var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); return LogEventBuilder.FromProperties(properties, remainder); } diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs new file mode 100644 index 00000000..47dc3448 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -0,0 +1,59 @@ +using System; +using System.Linq; +using SeqCli.PlainText.Patterns; +using Superpower; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class ExtractionPatternParserTests + { + [Fact] + public void ARegularStringIsASingleTextLiteral() + { + var pattern = ExtractionPatternParser.Parse("Hello!"); + Assert.Single(pattern.Elements); + var tt = Assert.IsType(pattern.Elements.Single()); + Assert.Equal("Hello!", tt.Text); + } + + [Fact] + public void CaptureNameAndTypeAreParsed() + { + var pattern = ExtractionPatternParser.Parse("{abc:def}"); + Assert.Single(pattern.Elements); + var ct = Assert.IsType(pattern.Elements.Single()); + Assert.Equal("abc", ct.Name); + Assert.Equal("def", ct.Type); + } + + [Theory] + [InlineData("", false)] + [InlineData("{}", false)] + [InlineData("{a", false)] + [InlineData("a", true)] + [InlineData("{a}", true)] + [InlineData("{@m}", true)] + [InlineData("{@m:n}", true)] + [InlineData("{@m:*}", true)] + [InlineData("{@m:n}", true)] + [InlineData("{m_N}", true)] + [InlineData("{_9}", true)] + [InlineData("{:n}", true)] + [InlineData("{:}", false)] + [InlineData("{{@m}}", true)] + [InlineData("{{a", true)] + [InlineData("a}}", true)] + [InlineData("{", false)] + [InlineData("}", false)] + [InlineData("{a} b{c} ", true)] + [InlineData("d {a}b {c}", true)] + public void OnlyValidPatternsAreAccepted(string attempt, bool isValid) + { + if (isValid) + ExtractionPatternParser.Parse(attempt); + else + Assert.Throws(() => ExtractionPatternParser.Parse(attempt)); + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/PatternTests.cs index e899cc4b..7d8a9d28 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/PatternTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using SeqCli.PlainText; +using SeqCli.PlainText.Extraction; using Superpower.Model; using Xunit; @@ -12,7 +13,7 @@ public class PatternTests public void TheDefaultPatternMatchesMultilineMessages() { var frame = $"Hello,{Environment.NewLine} world!"; - var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); Assert.Null(remainder); Assert.Single(properties, p => p.Key == ReifiedProperties.Message && ((TextSpan)p.Value).ToStringValue() == frame); @@ -22,16 +23,16 @@ public void TheDefaultPatternMatchesMultilineMessages() public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() { var frame = " world"; - var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); Assert.Empty(properties); Assert.Equal(frame, remainder); } - static Pattern ClassMethodPattern { get; } = new Pattern(new[] + static NameValueExtractor ClassMethodPattern { get; } = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "class"), - new PatternElement(BuiltInPatterns.LiteralText(".")), - new PatternElement(BuiltInPatterns.Identifier, "method") + new PatternElement(Matchers.Identifier, "class"), + new PatternElement(Matchers.LiteralText(".")), + new PatternElement(Matchers.Identifier, "method") }); [Fact] @@ -40,7 +41,7 @@ public void PatternsExtractElements() var pattern = ClassMethodPattern; var frame = "this.that"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("this", properties["class"].ToString()); Assert.Equal("that", properties["method"].ToString()); @@ -49,24 +50,24 @@ public void PatternsExtractElements() [Fact] public void TheFirstPatternElementIsExposed() { - Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); + Assert.Same(Matchers.Identifier, ClassMethodPattern.StartMarker); } [Fact] public void SingleLineContentMatchesUntilEol() { - var pattern = new Pattern(new[] + var pattern = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "first"), - new PatternElement(BuiltInPatterns.LiteralText(" ")), - new PatternElement(BuiltInPatterns.SingleLineContent, "content"), - new PatternElement(BuiltInPatterns.LiteralText(" (")), - new PatternElement(BuiltInPatterns.Identifier, "last"), - new PatternElement(BuiltInPatterns.LiteralText(")")) + new PatternElement(Matchers.Identifier, "first"), + new PatternElement(Matchers.LiteralText(" ")), + new PatternElement(Matchers.SingleLineContent, "content"), + new PatternElement(Matchers.LiteralText(" (")), + new PatternElement(Matchers.Identifier, "last"), + new PatternElement(Matchers.LiteralText(")")) }); var frame = "abc def ghi (jkl)"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("abc", properties["first"].ToString()); Assert.Equal("def ghi (jkl)", properties["content"].ToString()); @@ -79,20 +80,20 @@ public void NonGreedyContentStopsMatchingWhenFollowingTokensMatch() // the "following" list, since they effectively become "mandatory" var following = new[] { - new PatternElement(BuiltInPatterns.LiteralText(" (")), - new PatternElement(BuiltInPatterns.Identifier, "last"), - new PatternElement(BuiltInPatterns.LiteralText(")")) + new PatternElement(Matchers.LiteralText(" (")), + new PatternElement(Matchers.Identifier, "last"), + new PatternElement(Matchers.LiteralText(")")) }; - var pattern = new Pattern(new[] + var pattern = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "first"), - new PatternElement(BuiltInPatterns.LiteralText(" ")), - new PatternElement(BuiltInPatterns.NonGreedyContent(following), "content"), + new PatternElement(Matchers.Identifier, "first"), + new PatternElement(Matchers.LiteralText(" ")), + new PatternElement(Matchers.NonGreedyContent(following), "content"), }.Concat(following)); var frame = "abc def ghi (jkl)"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("abc", properties["first"].ToString()); Assert.Equal("def ghi", properties["content"].ToString()); From c424b98e991225a8039916b77924ee03ee4a75ed Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 23 Feb 2018 21:32:32 +1000 Subject: [PATCH 08/26] -p already taken for properties --- src/SeqCli/Cli/Commands/IngestCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index ed7817ae..900cec5a 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -47,7 +47,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); - Options.Add("p=|pattern=", + Options.Add("x=|extract=", "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)", v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); From 81c22a194db7c27a2b618edc9d90fa551733bea3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 07:31:42 +1000 Subject: [PATCH 09/26] Fix build status badge - don't show feature branches [Skip CI] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f9741e0b..170f7955 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# `seqcli` [![Build status](https://ci.appveyor.com/api/projects/status/sc3iacxwxqqfjgdh?svg=true)](https://ci.appveyor.com/project/datalust/seqcli) [![GitHub release](https://img.shields.io/github/release/datalust/seqcli.svg)](https://github.com/datalust/seqcli/releases) +# `seqcli` [![Build status](https://ci.appveyor.com/api/projects/status/sc3iacxwxqqfjgdh/branch/dev?svg=true)](https://ci.appveyor.com/project/datalust/seqcli/branch/dev) [![GitHub release](https://img.shields.io/github/release/datalust/seqcli.svg)](https://github.com/datalust/seqcli/releases) The [Seq](https://getseq.net) client command-line app. Supports logging (`seqcli log`), searching (`search`), tailing (`tail`), querying (`query`) and [JSON log file](https://github.com/serilog/serilog-formatting-compact) ingestion (`ingest`). From 7f1703c0dcbeecfcfcbbc0702ef12de40aca0c36 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 07:32:14 +1000 Subject: [PATCH 10/26] More placeholder tests, WIP --- src/SeqCli/PlainText/Extraction/Matchers.cs | 24 +++++++++++--- src/SeqCli/PlainText/Parsers/NumericsEx.cs | 21 +++++++++++++ src/SeqCli/PlainText/Parsers/StringsEx.cs | 16 ++++++++++ .../PlainText/PatternCompilerTests.cs | 31 +++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 src/SeqCli/PlainText/Parsers/NumericsEx.cs create mode 100644 src/SeqCli/PlainText/Parsers/StringsEx.cs create mode 100644 test/SeqCli.Tests/PlainText/PatternCompilerTests.cs diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 9f5c9fcc..0107057c 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -19,12 +19,27 @@ static class Matchers [Matcher("nat")] public static TextParser Natural { get; } = - Numerics.Natural + Numerics.NaturalUInt64 .Select(span => (object) span); [Matcher("int")] public static TextParser Integer { get; } = - Numerics.Integer + Numerics.IntegerInt64 + .Select(span => (object) span); + + [Matcher("dec")] + public static TextParser Decimal { get; } = + NumericsEx.Decimal + .Select(span => (object) span); + + [Matcher("alpha")] + public static TextParser Alphabetical { get; } = + Span.WithAll(char.IsLetter) + .Select(span => (object) span); + + [Matcher("alphanum")] + public static TextParser Alphanumeric { get; } = + Span.WithAll(char.IsLetterOrDigit) .Select(span => (object) span); [Matcher("token")] @@ -74,7 +89,8 @@ public static TextParser LiteralText(string literalText) public static TextParser NonGreedyContent(params PatternElement[] following) { if (following.Length == 0) - return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); + return SpanEx.MatchedBy(Character.AnyChar.Many()) + .Select(span => span.Length > 0 ? (object) span : null); var rest = following[0].Parser; for (var i = 1; i < following.Length; ++i) @@ -91,7 +107,7 @@ public static TextParser NonGreedyContent(params PatternElement[] follow } var span = i.Until(remainder); - return Result.Value((object) span, i, remainder); + return Result.Value(span.Length > 0 ? (object) span : null, i, remainder); }; } } diff --git a/src/SeqCli/PlainText/Parsers/NumericsEx.cs b/src/SeqCli/PlainText/Parsers/NumericsEx.cs new file mode 100644 index 00000000..1810372b --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/NumericsEx.cs @@ -0,0 +1,21 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class NumericsEx + { + public static TextParser Decimal { get; } = + Numerics.Integer + .Then(n => Character.EqualTo('.').IgnoreThen(Numerics.Integer).OptionalOrDefault() + .Select(f => f == TextSpan.None ? n : new TextSpan(n.Source, n.Position, n.Length + f.Length + 1))); + + public static TextParser HexNatural { get; } = + SpanEx.MatchedBy(Span.EqualTo("0x") + .IgnoreThen(Character.Digit + .Or(Character.Matching(ch => ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F', "a-f")) + .Named("hex digit") + .AtLeastOnce())); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Parsers/StringsEx.cs b/src/SeqCli/PlainText/Parsers/StringsEx.cs new file mode 100644 index 00000000..a39ffff9 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/StringsEx.cs @@ -0,0 +1,16 @@ +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class StringsEx + { + static readonly TextParser SqlStringContentChar = + Span.EqualTo("''").Value('\'').Try().Or(Character.ExceptIn('\'', '\r', '\n')); + + public static TextParser SqlStyle { get; } = + Character.EqualTo('\'') + .IgnoreThen(SqlStringContentChar.Many()) + .Then(s => Character.EqualTo('\'').Value(new string(s))); + } +} diff --git a/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs b/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs new file mode 100644 index 00000000..9aa4d9ab --- /dev/null +++ b/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs @@ -0,0 +1,31 @@ +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class PatternCompilerTests + { + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() + { + // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting + + var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:lines}"; + } + + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() + { + // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-console#output-templates + + var pattern = "[{@t:localtime} {@l:ident}] {@m:*}{:n}{@x:lines}"; + } + + [Fact] + public void OptionalSourceContextCanBeExtracted() + { + var pattern = "[{@t} {@l:ident}] ({SourceContext:*}) {@m:*}{:n}{@x:lines}"; + } + } +} \ No newline at end of file From cbe73366b53041ab27a821b0c6813563323bebfc Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 14:01:38 +1000 Subject: [PATCH 11/26] Enough working to parse default Serilog.Sinks.File formatted events --- ...der.cs => ExtractionPatternInterpreter.cs} | 11 ++- src/SeqCli/PlainText/Extraction/Matchers.cs | 19 ++++- .../PlainText/LogEvents/LogEventBuilder.cs | 18 +++-- src/SeqCli/PlainText/Parsers/DateTimesEx.cs | 12 +++ .../PlainText/PlainTextLogEventReader.cs | 4 +- .../ExtractionPatternInterpreterTests.cs | 74 +++++++++++++++++++ .../PlainText/LogEventBuilderTests.cs | 8 ++ ...ernTests.cs => NameValueExtractorTests.cs} | 6 +- .../PlainText/PatternCompilerTests.cs | 31 -------- 9 files changed, 135 insertions(+), 48 deletions(-) rename src/SeqCli/PlainText/Extraction/{PatternBuilder.cs => ExtractionPatternInterpreter.cs} (89%) create mode 100644 src/SeqCli/PlainText/Parsers/DateTimesEx.cs create mode 100644 test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs rename test/SeqCli.Tests/PlainText/{PatternTests.cs => NameValueExtractorTests.cs} (93%) delete mode 100644 test/SeqCli.Tests/PlainText/PatternCompilerTests.cs diff --git a/src/SeqCli/PlainText/Extraction/PatternBuilder.cs b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs similarity index 89% rename from src/SeqCli/PlainText/Extraction/PatternBuilder.cs rename to src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs index 0ae2aae7..68e7d302 100644 --- a/src/SeqCli/PlainText/Extraction/PatternBuilder.cs +++ b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs @@ -4,19 +4,20 @@ namespace SeqCli.PlainText.Extraction { - static class PatternCompiler + static class ExtractionPatternInterpreter { public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[] { new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) }); - public static NameValueExtractor Compile(ExtractionPattern pattern) + public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern) { if (pattern == null) throw new ArgumentNullException(nameof(pattern)); var patternElements = new PatternElement[pattern.Elements.Count]; - for (var i = 0; i < pattern.Elements.Count; ++i) + var last = true; + for (var i = pattern.Elements.Count - 1; i >= 0; --i) { var element = pattern.Elements[i]; switch (element) @@ -25,7 +26,7 @@ public static NameValueExtractor Compile(ExtractionPattern pattern) patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); break; case CapturePatternExpression capture when capture.Type == "*": - if (i < pattern.Elements.Count - 1) + if (!last) patternElements[i] = new PatternElement( Matchers.NonGreedyContent(patternElements[i + 1]), capture.Name); @@ -42,6 +43,8 @@ public static NameValueExtractor Compile(ExtractionPattern pattern) default: throw new InvalidOperationException($"Element `{element}` not recognized."); } + + last = false; } return new NameValueExtractor(patternElements); diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 0107057c..1b8232de 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Reflection; using SeqCli.PlainText.Parsers; @@ -46,6 +47,20 @@ static class Matchers public static TextParser Token { get; } = SpanEx.NonWhiteSpace.Select(span => (object)span); + [Matcher("iso8601dt")] + // A date and time are required by this pattern, though not necessarily by the spec. + public static TextParser Iso8601DateTime { get; } = + DateTimesEx.Iso8601DateTime + .Select(span => (object) span); + + public static TextParser SerilogFileTimestamp { get; } = + Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?") + .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture)); + + [Matcher("timestamp")] + public static TextParser Timestamp { get; } = + Iso8601DateTime.Try().Or(SerilogFileTimestamp); + // Unclear whether we need to name this public static TextParser MultiLineMessage { get; } = SpanEx.MatchedBy( @@ -101,9 +116,11 @@ public static TextParser NonGreedyContent(params PatternElement[] follow return i => { var remainder = i; - while (!rest.IsMatch(remainder)) + var attempt = rest(remainder); + while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything { remainder = remainder.ConsumeChar().Remainder; + attempt = rest(remainder); } var span = i.Until(remainder); diff --git a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs index f5b252c2..786b27cd 100644 --- a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs +++ b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs @@ -87,13 +87,17 @@ static IEnumerable GetLogEventProperties(IDictionary properties) { - var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && - t is TextSpan span && - DateTimeOffset.TryParse(span.ToStringValue(), CultureInfo.InvariantCulture, - DateTimeStyles.AssumeLocal, out var ts) - ? ts - : DateTimeOffset.Now; - return timestamp; + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; } static readonly Dictionary LevelsByName = new Dictionary(StringComparer.OrdinalIgnoreCase) diff --git a/src/SeqCli/PlainText/Parsers/DateTimesEx.cs b/src/SeqCli/PlainText/Parsers/DateTimesEx.cs new file mode 100644 index 00000000..7653505a --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/DateTimesEx.cs @@ -0,0 +1,12 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class DateTimesEx + { + public static TextParser Iso8601DateTime { get; } = + Span.Regex("\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?(([+-]\\d\\d:\\d\\d)|Z)?"); + } +} diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs index e6e68f28..af1cac3a 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -19,8 +19,8 @@ class PlainTextLogEventReader : ILogEventReader, IDisposable public PlainTextLogEventReader(TextReader input, string extractionPattern) { _nameValueExtractor = string.IsNullOrEmpty(extractionPattern) ? - PatternCompiler.MultilineMessageExtractor : - PatternCompiler.Compile(ExtractionPatternParser.Parse(extractionPattern)); + ExtractionPatternInterpreter.MultilineMessageExtractor : + ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); _reader = new FrameReader(input, SpanEx.MatchedBy(_nameValueExtractor.StartMarker), TrailingLineArrivalDeadline); } diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs new file mode 100644 index 00000000..6b555a83 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Globalization; +using SeqCli.PlainText; +using SeqCli.PlainText.Extraction; +using SeqCli.PlainText.Patterns; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class ExtractionPatternInterpreterTests + { + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() + { + // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting + + // {@l:ident} is required so that the default "token" pattern doesn't greedily eat up the `]`. + // "timestamp" is intended to be an aggregate timestamp parser that tries ISO 8601, RFC 2822, and various other + // popular timestamp formats. + + var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:*}"; + + var parsed = ExtractionPatternParser.Parse(pattern); + var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); + + var (properties, remainder) = extractor.ExtractValues( + @"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed +System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +"); + + Assert.Equal( + DateTimeOffset.ParseExact("2018-02-21 13:29:00.123 +10:00", "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture), + properties["@t"]); + Assert.Equal("ERR", properties["@l"].ToString()); + Assert.Equal("The operation failed", properties["@m"].ToString()); + Assert.Equal(@"System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +", properties["@x"].ToString()); + Assert.Null(remainder); + } + + // Work-in-progress... + + // [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() + { + // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-console#output-templates + + // "localtime" will add the closest non-future date to the time component that is matched + // by the pattern + + // The pattern language needs to be extended here so that the brackets, timestamp, spacing and + // level are all used as the start-frame marker. The strawman syntax proposes that to the + // right of `:` will always be either an alphanumeric matcher name, or a subexpression. This + // does have the issue that `{?:foo}` would be ambiguous (optional 'foo' matcher or optional 'foo' + // literal, so some escaping would be necessary - e.g. `{?:\foo}` to indicate a literal 'foo' and + // `{?:\*}` for an optional literal asterisk, `{?:\\}` for an optional literal backslash. + + var pattern = "{:[{@t:localtime} {@l:ident}] }{@m:*}{:n}{@x:*}"; + } + + // [Fact] + public void OptionalSourceContextCanBeExtracted() + { + // The {?: optional grouping is just an anonymous optional property, e.g. if the formatting was + // not dynamic, it might be written {SourceContext?:*}; using the grouping means the surrounding + // whitespace and parens are required only if the optional group is matched. + var pattern = "{:[{@t} {@l:ident}] }{?:({SourceContext:*}) }{@m:*}{:n}{@x:*}"; + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs index 50a67383..7cf38310 100644 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs @@ -45,5 +45,13 @@ public void MissingValuesAreDefaulted() Assert.Null(evt.Exception); Assert.Empty(evt.Properties); } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then, evt.Timestamp); + } } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs similarity index 93% rename from test/SeqCli.Tests/PlainText/PatternTests.cs rename to test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs index 7d8a9d28..785ede83 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs @@ -7,13 +7,13 @@ namespace SeqCli.Tests.PlainText { - public class PatternTests + public class NameValueExtractorTests { [Fact] public void TheDefaultPatternMatchesMultilineMessages() { var frame = $"Hello,{Environment.NewLine} world!"; - var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); + var (properties, remainder) = ExtractionPatternInterpreter.MultilineMessageExtractor.ExtractValues(frame); Assert.Null(remainder); Assert.Single(properties, p => p.Key == ReifiedProperties.Message && ((TextSpan)p.Value).ToStringValue() == frame); @@ -23,7 +23,7 @@ public void TheDefaultPatternMatchesMultilineMessages() public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() { var frame = " world"; - var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); + var (properties, remainder) = ExtractionPatternInterpreter.MultilineMessageExtractor.ExtractValues(frame); Assert.Empty(properties); Assert.Equal(frame, remainder); } diff --git a/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs b/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs deleted file mode 100644 index 9aa4d9ab..00000000 --- a/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Xunit; - -namespace SeqCli.Tests.PlainText -{ - public class PatternCompilerTests - { - [Fact] - public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() - { - // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" - // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting - - var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:lines}"; - } - - [Fact] - public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() - { - // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" - // See: https://github.com/serilog/serilog-sinks-console#output-templates - - var pattern = "[{@t:localtime} {@l:ident}] {@m:*}{:n}{@x:lines}"; - } - - [Fact] - public void OptionalSourceContextCanBeExtracted() - { - var pattern = "[{@t} {@l:ident}] ({SourceContext:*}) {@m:*}{:n}{@x:lines}"; - } - } -} \ No newline at end of file From 8634d6065af85880f2fac526e02ae4d94fab8f19 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sat, 24 Feb 2018 15:46:31 +1000 Subject: [PATCH 12/26] Add support for a subcommand --- src/SeqCli/Cli/CommandAttribute.cs | 6 ++++++ src/SeqCli/Cli/CommandLineHost.cs | 14 ++++++++++++-- src/SeqCli/Cli/CommandMetadata.cs | 1 + src/SeqCli/Cli/ICommandMetadata.cs | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/SeqCli/Cli/CommandAttribute.cs b/src/SeqCli/Cli/CommandAttribute.cs index a2b29b0b..81199286 100644 --- a/src/SeqCli/Cli/CommandAttribute.cs +++ b/src/SeqCli/Cli/CommandAttribute.cs @@ -20,6 +20,7 @@ namespace SeqCli.Cli public class CommandAttribute : Attribute, ICommandMetadata { public string Name { get; } + public string SubCommand { get; } public string HelpText { get; } public string Example { get; set; } @@ -29,5 +30,10 @@ public CommandAttribute(string name, string helpText) Name = name; HelpText = helpText; } + + public CommandAttribute(string name, string subCommand, string helpText) : this(name, helpText) + { + SubCommand = subCommand; + } } } diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index 2b635f1c..0a035767 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -37,11 +37,21 @@ public async Task Run(string[] args) if (args.Length > 0) { + var amountToSkip = 1; var norm = args[0].ToLowerInvariant(); - var cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm); + Meta, CommandMetadata> cmd; + if (!args[1].Contains("-")) + { + amountToSkip = 2; + cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm && c.Metadata.SubCommand == args[1].ToLowerInvariant()); + } + else + { + cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm); + } if (cmd != null) { - return await cmd.Value.Value.Invoke(args.Skip(1).ToArray()); + return await cmd.Value.Value.Invoke(args.Skip(amountToSkip).ToArray()); } } diff --git a/src/SeqCli/Cli/CommandMetadata.cs b/src/SeqCli/Cli/CommandMetadata.cs index acf71400..f1bd8fe7 100644 --- a/src/SeqCli/Cli/CommandMetadata.cs +++ b/src/SeqCli/Cli/CommandMetadata.cs @@ -17,6 +17,7 @@ namespace SeqCli.Cli public class CommandMetadata : ICommandMetadata { public string Name { get; set; } + public string SubCommand { get; set; } public string HelpText { get; set; } public string Example { get; set; } } diff --git a/src/SeqCli/Cli/ICommandMetadata.cs b/src/SeqCli/Cli/ICommandMetadata.cs index b5f653d5..7d4d0c4b 100644 --- a/src/SeqCli/Cli/ICommandMetadata.cs +++ b/src/SeqCli/Cli/ICommandMetadata.cs @@ -17,6 +17,7 @@ namespace SeqCli.Cli interface ICommandMetadata { string Name { get; } + string SubCommand { get; } string HelpText { get; } } } From 46a0617b8ca4ff73c33051879a3b3b1b9d2da9d6 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sat, 24 Feb 2018 16:17:13 +1000 Subject: [PATCH 13/26] Only pick up main command if no subcommand specified --- src/SeqCli/Cli/CommandLineHost.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index 0a035767..3e93033a 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -47,7 +47,7 @@ public async Task Run(string[] args) } else { - cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm); + cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm && c.Metadata.SubCommand == default); } if (cmd != null) { From bdae353bcee93335913297a26eb216161dbeb0df Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sat, 24 Feb 2018 16:40:25 +1000 Subject: [PATCH 14/26] Add test to ensure no commands using the same name and subcommand --- test/SeqCli.Tests/Cli/NoDuplicateCommands.cs | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 test/SeqCli.Tests/Cli/NoDuplicateCommands.cs diff --git a/test/SeqCli.Tests/Cli/NoDuplicateCommands.cs b/test/SeqCli.Tests/Cli/NoDuplicateCommands.cs new file mode 100644 index 00000000..532ddef5 --- /dev/null +++ b/test/SeqCli.Tests/Cli/NoDuplicateCommands.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using SeqCli.Cli; +using Xunit; + +namespace SeqCli.Tests.Cli +{ + + public class NoDuplicateCommands + { + [Fact] + public void EnsureNoDuplicateCommands() + { + var anyDuplicates = typeof(Command).GetTypeInfo().Assembly.GetExportedTypes() + .Where(t => t.IsAssignableFrom(typeof(Command))) + .Select(t => new {CommandType = t, Attribute = t.GetCustomAttribute()}) + .GroupBy(t => new {t.Attribute.Name, t.Attribute.SubCommand}) + .Any(t => t.Count() > 1); + + Assert.Equal(anyDuplicates, false); + } + } +} From cf1647d74e6bd0cb4f55f7d8724d7611ab944929 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 20:15:23 +1000 Subject: [PATCH 15/26] Multiple-token non-greedy lookahead --- .../ExtractionPatternInterpreter.cs | 23 +++++-------- src/SeqCli/PlainText/Extraction/Matchers.cs | 7 +++- .../Patterns/CaptureContentExpression.cs | 6 ++++ .../Patterns/CapturePatternExpression.cs | 12 +++---- .../Patterns/ExtractionPatternParser.cs | 16 ++++----- .../Patterns/MatchTypeContentExpression.cs | 12 +++++++ .../Patterns/NonGreedyContentExpression.cs | 12 +++++++ .../ExtractionPatternInterpreterTests.cs | 33 +++++++++++++++---- .../PlainText/ExtractionPatternParserTests.cs | 7 +++- 9 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs diff --git a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs index 68e7d302..16834b93 100644 --- a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs +++ b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using SeqCli.PlainText.Patterns; namespace SeqCli.PlainText.Extraction @@ -16,7 +17,6 @@ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern patt if (pattern == null) throw new ArgumentNullException(nameof(pattern)); var patternElements = new PatternElement[pattern.Elements.Count]; - var last = true; for (var i = pattern.Elements.Count - 1; i >= 0; --i) { var element = pattern.Elements[i]; @@ -25,26 +25,21 @@ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern patt case LiteralTextPatternExpression text: patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); break; - case CapturePatternExpression capture when capture.Type == "*": - if (!last) - patternElements[i] = new PatternElement( - Matchers.NonGreedyContent(patternElements[i + 1]), - capture.Name); - else - patternElements[i] = new PatternElement( - Matchers.NonGreedyContent(), // <- same as MultiLineContent - capture.Name); + case CapturePatternExpression capture + when capture.Content is NonGreedyContentExpression ngc: + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()), + capture.Name); break; - case CapturePatternExpression capture: + case CapturePatternExpression capture + when capture.Content is MatchTypeContentExpression mtc: patternElements[i] = new PatternElement( - capture.Type == null ? Matchers.Token : Matchers.GetByType(capture.Type), + mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type), capture.Name); break; default: throw new InvalidOperationException($"Element `{element}` not recognized."); } - - last = false; } return new NameValueExtractor(patternElements); diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 1b8232de..ec2725bd 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -68,7 +68,7 @@ static class Matchers .IgnoreThen(Character.AnyChar.Many())) .Select(span => (object)span); - [Matcher("lines")] + // Equivalent to :* at end-of-pattern public static TextParser MultiLineContent { get; } = Span.WithAll(ch => true) .Select(span => (object)span); @@ -84,6 +84,11 @@ from _ in NewLine.OptionalOrDefault() Span.EqualTo("\r\n").Or(Span.EqualTo("\n")) .Select(span => (object)span); + [Matcher("t")] + public static TextParser Tab { get; } = + Span.EqualTo("\t") + .Select(span => (object)span); + static readonly Dictionary> ByType = new Dictionary>( from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties let attr = pi.GetCustomAttribute() diff --git a/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs b/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs new file mode 100644 index 00000000..ef4d8ec1 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs @@ -0,0 +1,6 @@ +namespace SeqCli.PlainText.Patterns +{ + abstract class CaptureContentExpression + { + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs index def5648d..4e76bca5 100644 --- a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs +++ b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs @@ -1,16 +1,14 @@ -using System; - -namespace SeqCli.PlainText.Patterns +namespace SeqCli.PlainText.Patterns { class CapturePatternExpression : ExtractionPatternExpression { public string Name { get; } - public string Type { get; } + public CaptureContentExpression Content { get; } - public CapturePatternExpression(string name, string type) + public CapturePatternExpression(string name, CaptureContentExpression content) { Name = name; - Type = type; + Content = content; } } -} \ No newline at end of file +} diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs index 1038967a..493edb1d 100644 --- a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -20,21 +20,21 @@ static class ExtractionPatternParser .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) .Select(s => s.ToStringValue()); - static readonly TextParser CaptureType = - SpanEx.MatchedBy(Character.EqualTo('*')) + static readonly TextParser CaptureContent = + Character.EqualTo('*').AtLeastOnce().Select(chs => (CaptureContentExpression)new NonGreedyContentExpression(chs.Length)) .Or(SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) - .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many()))) - .Select(s => s.ToStringValue()); + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => (CaptureContentExpression)new MatchTypeContentExpression(s.ToStringValue()))); static readonly TextParser Capture = from _ in Character.EqualTo('{') from name in CaptureName.OptionalOrDefault() - from type in Character.EqualTo(':') - .IgnoreThen(CaptureType) + from content in Character.EqualTo(':') + .IgnoreThen(CaptureContent) .OptionalOrDefault() - where name != null || type != null + where name != null || content != null from __ in Character.EqualTo('}') - select new CapturePatternExpression(name, type); + select new CapturePatternExpression(name, content); static readonly TextParser Element = LiteralText.Cast() diff --git a/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs b/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs new file mode 100644 index 00000000..1a0f9bf9 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs @@ -0,0 +1,12 @@ +namespace SeqCli.PlainText.Patterns +{ + class MatchTypeContentExpression : CaptureContentExpression + { + public string Type { get; } + + public MatchTypeContentExpression(string type) + { + Type = type; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs b/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs new file mode 100644 index 00000000..bb4a2969 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs @@ -0,0 +1,12 @@ +namespace SeqCli.PlainText.Patterns +{ + class NonGreedyContentExpression : CaptureContentExpression + { + public int Lookahead { get; } + + public NonGreedyContentExpression(int lookahead) + { + Lookahead = lookahead; + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs index 6b555a83..f14b1b77 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using SeqCli.PlainText; using SeqCli.PlainText.Extraction; @@ -9,6 +10,21 @@ namespace SeqCli.Tests.PlainText { public class ExtractionPatternInterpreterTests { + static (IDictionary, string) ExtractValues(string pattern, string candidate) + { + var parsed = ExtractionPatternParser.Parse(pattern); + var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); + return extractor.ExtractValues(candidate); + } + + [Fact] + public void NonGreedyMatchCanLookaheadMultipleTokens() + { + var (properties, remainder) = ExtractValues("[{test:**}]!", "[0]abc[1]!"); + Assert.Null(remainder); + Assert.Equal("0]abc[1", properties["test"].ToString()); + } + [Fact] public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() { @@ -20,15 +36,14 @@ public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() // popular timestamp formats. var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:*}"; - - var parsed = ExtractionPatternParser.Parse(pattern); - var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); - - var (properties, remainder) = extractor.ExtractValues( - @"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed + + var candidate = +@"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed System.DivideByZeroException: Attempt to divide by zero at SomeClass.SomeMethod() -"); +"; + + var (properties, remainder) = ExtractValues(pattern, candidate); Assert.Equal( DateTimeOffset.ParseExact("2018-02-21 13:29:00.123 +10:00", "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture), @@ -59,7 +74,9 @@ public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() // literal, so some escaping would be necessary - e.g. `{?:\foo}` to indicate a literal 'foo' and // `{?:\*}` for an optional literal asterisk, `{?:\\}` for an optional literal backslash. +#pragma warning disable 219 var pattern = "{:[{@t:localtime} {@l:ident}] }{@m:*}{:n}{@x:*}"; +#pragma warning restore 219 } // [Fact] @@ -68,7 +85,9 @@ public void OptionalSourceContextCanBeExtracted() // The {?: optional grouping is just an anonymous optional property, e.g. if the formatting was // not dynamic, it might be written {SourceContext?:*}; using the grouping means the surrounding // whitespace and parens are required only if the optional group is matched. +#pragma warning disable 219 var pattern = "{:[{@t} {@l:ident}] }{?:({SourceContext:*}) }{@m:*}{:n}{@x:*}"; +#pragma warning restore 219 } } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 47dc3448..85fd84fc 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -24,7 +24,7 @@ public void CaptureNameAndTypeAreParsed() Assert.Single(pattern.Elements); var ct = Assert.IsType(pattern.Elements.Single()); Assert.Equal("abc", ct.Name); - Assert.Equal("def", ct.Type); + Assert.Equal("def", ((MatchTypeContentExpression)ct.Content).Type); } [Theory] @@ -34,8 +34,13 @@ public void CaptureNameAndTypeAreParsed() [InlineData("a", true)] [InlineData("{a}", true)] [InlineData("{@m}", true)] + [InlineData("{@@m}", false)] + [InlineData("{m@}", false)] [InlineData("{@m:n}", true)] [InlineData("{@m:*}", true)] + [InlineData("{@m:***}", true)] + [InlineData("{:*}", true)] + [InlineData("{a:}", false)] [InlineData("{@m:n}", true)] [InlineData("{m_N}", true)] [InlineData("{_9}", true)] From 929ef8a5eb41726fd1ade26cb0babcb6f4ebb6bb Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sun, 25 Feb 2018 09:59:56 +1000 Subject: [PATCH 16/26] Cleaned up subcommand finding code --- src/SeqCli/Cli/CommandLineHost.cs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index 3e93033a..bc725e3c 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -37,20 +37,12 @@ public async Task Run(string[] args) if (args.Length > 0) { - var amountToSkip = 1; var norm = args[0].ToLowerInvariant(); - Meta, CommandMetadata> cmd; - if (!args[1].Contains("-")) - { - amountToSkip = 2; - cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm && c.Metadata.SubCommand == args[1].ToLowerInvariant()); - } - else - { - cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm && c.Metadata.SubCommand == default); - } + var subCommandNorm = args.Length > 1 && !args[1].Contains("-") ? args[1].ToLowerInvariant() : default; + var cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm && c.Metadata.SubCommand == subCommandNorm); if (cmd != null) { + var amountToSkip = subCommandNorm == default ? 1 : 2; return await cmd.Value.Value.Invoke(args.Skip(amountToSkip).ToArray()); } } From 1b19c0444744f496c779246f59d459e96f8067f0 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sun, 25 Feb 2018 10:22:45 +1000 Subject: [PATCH 17/26] Add some tests to ensure correct command and subcommands are picked --- test/SeqCli.Tests/Cli/CommandLineHostTests.cs | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 test/SeqCli.Tests/Cli/CommandLineHostTests.cs diff --git a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs new file mode 100644 index 00000000..049c24df --- /dev/null +++ b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Autofac.Features.Metadata; +using SeqCli.Cli; +using SeqCli.Cli.Commands; +using Xunit; + +namespace SeqCli.Tests.Cli +{ + public class CommandLineHostTests + { + [Fact] + public async Task CheckCommandLineHostPicksCorrectCommand() + { + var commandsRan = new List(); + var availableCommands = new List, CommandMetadata>> + { + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test"))), + new CommandMetadata() {Name = "test"}), + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test2"))), + new CommandMetadata() {Name = "test2"}) + }; + var commandLineHost = new CommandLineHost(availableCommands); + await commandLineHost.Run(new []{ "test"}); + + Assert.Equal(commandsRan.First(), "test"); + } + + [Fact] + public async Task WhenCommandAndSubcommandAndTheUserRunsWithoutSubcommandEnsurePickedCorrect() + { + var commandsRan = new List(); + var availableCommands = + new List, CommandMetadata>> + { + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test"))), + new CommandMetadata() {Name = "test"}), + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand"))), + new CommandMetadata() {Name = "test", SubCommand = "subcommand"}) + }; + var commandLineHost = new CommandLineHost(availableCommands); + await commandLineHost.Run(new[] { "test" }); + + Assert.Equal(commandsRan.First(), "test"); + } + + [Fact] + public async Task WhenCommandAndSubcommandAndTheUserRunsWithSubcommandEnsurePickedCorrect() + { + var commandsRan = new List(); + var availableCommands = + new List, CommandMetadata>> + { + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test"))), + new CommandMetadata() {Name = "test"}), + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand"))), + new CommandMetadata() {Name = "test", SubCommand = "subcommand"}) + }; + var commandLineHost = new CommandLineHost(availableCommands); + await commandLineHost.Run(new[] { "test", "subcommand" }); + + Assert.Equal(commandsRan.First(), "test-subcommand"); + } + + [Fact] + public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePickedCorrect() + { + var commandsRan = new List(); + var availableCommands = + new List, CommandMetadata>> + { + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand1"))), + new CommandMetadata() {Name = "test", SubCommand = "subcommand1"}), + new Meta, CommandMetadata>( + new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand2"))), + new CommandMetadata() {Name = "test", SubCommand = "subcommand2"}) + }; + var commandLineHost = new CommandLineHost(availableCommands); + await commandLineHost.Run(new[] { "test", "subcommand2" }); + + Assert.Equal(commandsRan.First(), "test-subcommand2"); + } + + class ActionCommand : Command + { + public ActionCommand(Action action) + { + action.Invoke(); + } + } + } +} From 5bf6c18534e7f5123279f32f9159686c7e622c2f Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sun, 25 Feb 2018 20:20:29 +1000 Subject: [PATCH 18/26] Groups work-in-progress --- .../ExtractionPatternInterpreter.cs | 64 +++++++++---------- .../Extraction/GroupedPatternElement.cs | 51 +++++++++++++++ src/SeqCli/PlainText/Extraction/Matchers.cs | 4 +- .../Extraction/NameValueExtractor.cs | 12 +--- .../PlainText/Extraction/PatternElement.cs | 31 ++++++--- .../Extraction/SimplePatternElement.cs | 39 +++++++++++ .../Patterns/ExtractionPatternParser.cs | 25 ++++++-- .../Patterns/GroupedContentExpression.cs | 14 ++++ .../PlainText/ExtractionPatternParserTests.cs | 3 + .../PlainText/NameValueExtractorTests.cs | 32 +++++----- 10 files changed, 201 insertions(+), 74 deletions(-) create mode 100644 src/SeqCli/PlainText/Extraction/GroupedPatternElement.cs create mode 100644 src/SeqCli/PlainText/Extraction/SimplePatternElement.cs create mode 100644 src/SeqCli/PlainText/Patterns/GroupedContentExpression.cs diff --git a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs index 16834b93..7883d668 100644 --- a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs +++ b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs @@ -9,10 +9,10 @@ static class ExtractionPatternInterpreter { public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[] { - new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) + new SimplePatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) }); - public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern) + static PatternElement[] CreatePatternElements(ExtractionPattern pattern) { if (pattern == null) throw new ArgumentNullException(nameof(pattern)); @@ -22,39 +22,39 @@ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern patt var element = pattern.Elements[i]; switch (element) { - case LiteralTextPatternExpression text: - patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); - break; - case CapturePatternExpression capture - when capture.Content is NonGreedyContentExpression ngc: - patternElements[i] = new PatternElement( - Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()), - capture.Name); - break; - case CapturePatternExpression capture - when capture.Content is MatchTypeContentExpression mtc: - patternElements[i] = new PatternElement( - mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type), - capture.Name); - break; - default: - throw new InvalidOperationException($"Element `{element}` not recognized."); + case LiteralTextPatternExpression text: + patternElements[i] = new SimplePatternElement(Matchers.LiteralText(text.Text)); + break; + case CapturePatternExpression capture + when capture.Content is NonGreedyContentExpression ngc: + patternElements[i] = new SimplePatternElement( + Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()), + capture.Name); + break; + case CapturePatternExpression capture + when capture.Content is MatchTypeContentExpression mtc: + patternElements[i] = new SimplePatternElement( + mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type), + capture.Name); + break; + case CapturePatternExpression capture + when capture.Content is GroupedContentExpression gc: + patternElements[i] = new GroupedPatternElement( + CreatePatternElements(gc.ExtractionPattern), + capture.Name); + break; + default: + throw new InvalidOperationException($"Element `{element}` not recognized."); } } - - return new NameValueExtractor(patternElements); + + return patternElements; } - // What we need to do here is: - // - for each parsed token - // - if it's literal text, map it an anonymous PatternElement with - // BuiltInPatterns.LiteralText() - // - otherwise, if it specifies no format, it's a named element with - // the BuiltInPatterns.Token parser - // - if it does specify a format, look up the parser based on the name, except - // - if the format is `$` it is BuiltInPatterns.SingleLineContent - // - if the format is `$$` it is BuiltInPatterns.MultiLineContent - // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the - // parser that follows it + public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern) + { + var patternElements = CreatePatternElements(pattern); + return new NameValueExtractor(patternElements); + } } } \ No newline at end of file diff --git a/src/SeqCli/PlainText/Extraction/GroupedPatternElement.cs b/src/SeqCli/PlainText/Extraction/GroupedPatternElement.cs new file mode 100644 index 00000000..d269cfca --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/GroupedPatternElement.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Superpower; +using Superpower.Model; + +namespace SeqCli.PlainText.Extraction +{ + class GroupedPatternElement : PatternElement + { + readonly PatternElement[] _content; + + public GroupedPatternElement(IEnumerable content, string name = null) + : base(name) + { + _content = content?.ToArray() ?? throw new ArgumentNullException(nameof(content)); + if (_content.Length == 0) throw new ArgumentException("A grouped pattern must include at least one element."); + + Match = _content.Select(c => c.Match).Aggregate((a, b) => a.IgnoreThen(b)); + } + + public override TextParser Match { get; } + + public override bool TryExtract( + TextSpan input, + Dictionary result, + out TextSpan remainder) + { + var temp = new Dictionary(); + + var rem = input; + foreach (var element in _content) + { + if (!element.TryExtract(rem, temp, out rem)) + { + remainder = input; + return false; + } + } + + foreach (var pair in temp) + result.Add(pair.Key, pair.Value); + + var value = input.Until(rem); + remainder = rem; + CollectResult(result, value); + + return true; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index ec2725bd..1eb482a6 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -112,10 +112,10 @@ public static TextParser NonGreedyContent(params PatternElement[] follow return SpanEx.MatchedBy(Character.AnyChar.Many()) .Select(span => span.Length > 0 ? (object) span : null); - var rest = following[0].Parser; + var rest = following[0].Match; for (var i = 1; i < following.Length; ++i) { - rest = rest.IgnoreThen(following[i].Parser); + rest = rest.IgnoreThen(following[i].Match); } return i => diff --git a/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs index 243fe6a3..c1933c98 100644 --- a/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs +++ b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs @@ -18,7 +18,7 @@ public NameValueExtractor(IEnumerable elements) throw new ArgumentException("An extraction pattern must contain at least one element."); } - public TextParser StartMarker => _elements[0].Parser; + public TextParser StartMarker => _elements[0].Match; public (IDictionary, string) ExtractValues(string plainText) { @@ -28,21 +28,13 @@ public NameValueExtractor(IEnumerable elements) var remainder = input; foreach (var element in _elements) { - var match = element.Parser(remainder); - if (!match.HasValue) + if (!element.TryExtract(remainder, result, out remainder)) { if (remainder.IsAtEnd || Span.WhiteSpace.IsMatch(remainder)) return (result, null); return (result, remainder.ToStringValue()); } - - remainder = match.Remainder; - - if (!element.IsIgnored) - { - result.Add(element.Name, match.Value); - } } return (result, null); diff --git a/src/SeqCli/PlainText/Extraction/PatternElement.cs b/src/SeqCli/PlainText/Extraction/PatternElement.cs index 29670dd9..42412afd 100644 --- a/src/SeqCli/PlainText/Extraction/PatternElement.cs +++ b/src/SeqCli/PlainText/Extraction/PatternElement.cs @@ -1,18 +1,31 @@ -using System; +using System.Collections.Generic; using Superpower; +using Superpower.Model; namespace SeqCli.PlainText.Extraction { - class PatternElement + abstract class PatternElement { - public PatternElement(TextParser parser, string name = null) + readonly string _name; + + bool IsIgnored => _name == null; + + protected PatternElement(string name) { - Parser = parser ?? throw new ArgumentNullException(nameof(parser)); - Name = name; + _name = name; } - public TextParser Parser { get; } - public string Name { get; } - public bool IsIgnored => Name == null; + public abstract TextParser Match { get; } + + public abstract bool TryExtract( + TextSpan input, + Dictionary result, + out TextSpan remainder); + + protected void CollectResult(Dictionary result, object value) + { + if (!IsIgnored) + result.Add(_name, value); + } } -} \ No newline at end of file +} diff --git a/src/SeqCli/PlainText/Extraction/SimplePatternElement.cs b/src/SeqCli/PlainText/Extraction/SimplePatternElement.cs new file mode 100644 index 00000000..46ef94be --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/SimplePatternElement.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using Superpower; +using Superpower.Model; + +namespace SeqCli.PlainText.Extraction +{ + class SimplePatternElement : PatternElement + { + readonly TextParser _parser; + + public override TextParser Match { get; } + + public SimplePatternElement(TextParser parser, string name = null) + : base(name) + { + _parser = parser ?? throw new ArgumentNullException(nameof(parser)); + Match = _parser.Select(s => Unit.Value); + } + + public override bool TryExtract( + TextSpan input, + Dictionary result, + out TextSpan remainder) + { + var match = _parser(input); + if (!match.HasValue) + { + remainder = input; + return false; + } + + CollectResult(result, match.Value); + remainder = match.Remainder; + + return true; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs index 493edb1d..f0b13e5e 100644 --- a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -20,11 +20,23 @@ static class ExtractionPatternParser .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) .Select(s => s.ToStringValue()); + static readonly TextParser NonGreedyContent = + Character.EqualTo('*').AtLeastOnce() + .Select(chs => (CaptureContentExpression) new NonGreedyContentExpression(chs.Length)); + + static readonly TextParser MatchTypeContent = + SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => (CaptureContentExpression) new MatchTypeContentExpression(s.ToStringValue())); + + static readonly TextParser GroupedContent = + Superpower.Parse.Ref(() => Elements) + .Select(els => (CaptureContentExpression) new GroupedContentExpression(new ExtractionPattern(els))); + static readonly TextParser CaptureContent = - Character.EqualTo('*').AtLeastOnce().Select(chs => (CaptureContentExpression)new NonGreedyContentExpression(chs.Length)) - .Or(SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) - .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) - .Select(s => (CaptureContentExpression)new MatchTypeContentExpression(s.ToStringValue()))); + NonGreedyContent + .Or(MatchTypeContent).Try() + .Or(GroupedContent); static readonly TextParser Capture = from _ in Character.EqualTo('{') @@ -40,8 +52,11 @@ from __ in Character.EqualTo('}') LiteralText.Cast() .Or(Capture.Cast()); + static readonly TextParser Elements = + Element.AtLeastOnce(); + static readonly TextParser Pattern = - Element.AtLeastOnce().AtEnd().Select(e => new ExtractionPattern(e)); + Elements.AtEnd().Select(e => new ExtractionPattern(e)); public static ExtractionPattern Parse(string extractionPattern) { diff --git a/src/SeqCli/PlainText/Patterns/GroupedContentExpression.cs b/src/SeqCli/PlainText/Patterns/GroupedContentExpression.cs new file mode 100644 index 00000000..23cf2f33 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/GroupedContentExpression.cs @@ -0,0 +1,14 @@ +using System; + +namespace SeqCli.PlainText.Patterns +{ + class GroupedContentExpression : CaptureContentExpression + { + public ExtractionPattern ExtractionPattern { get; } + + public GroupedContentExpression(ExtractionPattern extractionPattern) + { + ExtractionPattern = extractionPattern ?? throw new ArgumentNullException(nameof(extractionPattern)); + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 85fd84fc..03cecae2 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -53,6 +53,9 @@ public void CaptureNameAndTypeAreParsed() [InlineData("}", false)] [InlineData("{a} b{c} ", true)] [InlineData("d {a}b {c}", true)] + [InlineData("{:{@m}}", true)] + [InlineData("Loaded {SignalId:signal-{:nat}}", true)] + [InlineData("{:{Year:num}-{Month:num}}", true)] public void OnlyValidPatternsAreAccepted(string attempt, bool isValid) { if (isValid) diff --git a/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs index 785ede83..38fc85ac 100644 --- a/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs +++ b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs @@ -30,9 +30,9 @@ public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() static NameValueExtractor ClassMethodPattern { get; } = new NameValueExtractor(new[] { - new PatternElement(Matchers.Identifier, "class"), - new PatternElement(Matchers.LiteralText(".")), - new PatternElement(Matchers.Identifier, "method") + new SimplePatternElement(Matchers.Identifier, "class"), + new SimplePatternElement(Matchers.LiteralText(".")), + new SimplePatternElement(Matchers.Identifier, "method") }); [Fact] @@ -50,7 +50,7 @@ public void PatternsExtractElements() [Fact] public void TheFirstPatternElementIsExposed() { - Assert.Same(Matchers.Identifier, ClassMethodPattern.StartMarker); + Assert.NotNull(ClassMethodPattern.StartMarker); } [Fact] @@ -58,12 +58,12 @@ public void SingleLineContentMatchesUntilEol() { var pattern = new NameValueExtractor(new[] { - new PatternElement(Matchers.Identifier, "first"), - new PatternElement(Matchers.LiteralText(" ")), - new PatternElement(Matchers.SingleLineContent, "content"), - new PatternElement(Matchers.LiteralText(" (")), - new PatternElement(Matchers.Identifier, "last"), - new PatternElement(Matchers.LiteralText(")")) + new SimplePatternElement(Matchers.Identifier, "first"), + new SimplePatternElement(Matchers.LiteralText(" ")), + new SimplePatternElement(Matchers.SingleLineContent, "content"), + new SimplePatternElement(Matchers.LiteralText(" (")), + new SimplePatternElement(Matchers.Identifier, "last"), + new SimplePatternElement(Matchers.LiteralText(")")) }); var frame = "abc def ghi (jkl)"; @@ -80,16 +80,16 @@ public void NonGreedyContentStopsMatchingWhenFollowingTokensMatch() // the "following" list, since they effectively become "mandatory" var following = new[] { - new PatternElement(Matchers.LiteralText(" (")), - new PatternElement(Matchers.Identifier, "last"), - new PatternElement(Matchers.LiteralText(")")) + new SimplePatternElement(Matchers.LiteralText(" (")), + new SimplePatternElement(Matchers.Identifier, "last"), + new SimplePatternElement(Matchers.LiteralText(")")) }; var pattern = new NameValueExtractor(new[] { - new PatternElement(Matchers.Identifier, "first"), - new PatternElement(Matchers.LiteralText(" ")), - new PatternElement(Matchers.NonGreedyContent(following), "content"), + new SimplePatternElement(Matchers.Identifier, "first"), + new SimplePatternElement(Matchers.LiteralText(" ")), + new SimplePatternElement(Matchers.NonGreedyContent(following), "content"), }.Concat(following)); var frame = "abc def ghi (jkl)"; From 36f8a318d30b60573da67e8d36c7a632498b3659 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Mon, 26 Feb 2018 20:28:35 +1000 Subject: [PATCH 19/26] Equals for compound match expressions --- README.md | 60 ++++++++++++++++++- .../Patterns/ExtractionPatternParser.cs | 5 +- .../PlainText/ExtractionPatternParserTests.cs | 6 +- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f9741e0b..5fb4e25b 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Send JSON log events from a file or `STDIN`. Example: ``` -seqcli ingest -i events.clef --filter="@Level <> 'Debug'" -p Environment=Test +seqcli ingest -i events.clef --json --filter="@Level <> 'Debug'" -p Environment=Test ``` | Option | Description | @@ -63,6 +63,8 @@ seqcli ingest -i events.clef --filter="@Level <> 'Debug'" -p Environment=Test | `-i`, `--input=VALUE` | CLEF file to ingest; if not specified, `STDIN` will be used | | `--invalid-data=VALUE` | Specify how invalid data is handled: fail (default) or ignore | | `-p`, `--property=VALUE1=VALUE2` | Specify event properties, e.g. `-p Customer=C123 -p Environment=Production` | +| `-x`, `--extract=VALUE` | An extraction pattern to apply to plain-text logs (ignored when `--json` is specified) | +| `--json` | Read the events as JSON (the default assumes plain text) | | `-f`, `--filter=VALUE` | Filter expression to select a subset of events | | `-s`, `--server=VALUE` | The URL of the Seq server; by default the `connection.serverUrl` value will be used | | `-a`, `--apikey=VALUE` | The API key to use when connecting to the server; by default `config.apiKey` value will be used | @@ -147,3 +149,59 @@ Stream log events matching a filter. ### `version` Print the current executable version. + +## Extraction Patterns + +The `seqcli ingest` command can be used for parsing plain text logs into structured log events. + +```shell +seqcli ingest -x "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:*}" +``` + +The `-x` argument above is an _extraction pattern_ that will parse events like: + +``` +2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed +System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +``` + +### Syntax + +Extraction patterns have a simple high-level syntax: + + * Text that appears in the pattern is matched literally - so a pattern like `Hello, world!` will match logging statements that are made up of this greeting only, + * Text between `{curly braces}` is a _match expression_ that identifies a part of the event to be extracted, and + * Literal curly braces are escaped by doubling, so `{{` will match the literal text `{`, and `}}` matches `}`. + +Match expressions have the syntax: + +``` +{name:matcher} +``` + +Both the name and matcher are optional, but either one or the other must be specified. Hence `{@t:timestamp}` specifies a name of `@t` and value `timestamp`, `{IPAddress}` specifies a name only, and `{:n}` a value only (in this case the built-in newline matcher). + +The _name_ is the property name to be extracted; there are four built-in property names that get special handling: + + * `@t` - the event's timestamp + * `@m` - the textual message associated with the event + * `@l` - the event's level + * `@x` - the exception or backtrace associated with the event + +Other property names are attached to the event payload, so `{Elapsed:dec}` will extract a property called `Elapsed`, using the `dec` decimal matcher. + +Match expressions with no name are consumed from the input, but are not added to the event payload. + +### Matchers + +Matchers identify chunks of the input event. + +Different matchers are needed so that a piece of text like `200OK` can be separated into separate properties, i.e. `{StatusCode:nat}{Status:alpha}`. Here, the `nat` (natural number) matcher also coerces the result into a numeric value, so that it is attached to the event payload numerically as `200` instead of as the text `"200"`. + +There are three kinds of matchers: + + * Matchers like `alpha` and `nat` are built-in _named_ matchers. These are built-in. + * The special matchers `*`, `**` and so-on, are _non-greedy content_ matchers; these will match any text up until the next pattern element matches (`*`), the next two elements match, and so-on. We saw this in action with the `{@m:*}{:n}` elements in the example - the message is all of the text up until the next newline. + * More complex _compound_ matchers are described using a sub-expression. These are prefixed with an equals sign `=`, like `{Phone:={:nat}-{:nat}-{:nat}}`. This will extract chunks of text like `123-456-7890` into the `Phone` property. + \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs index f0b13e5e..ff3a2145 100644 --- a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -30,12 +30,13 @@ static class ExtractionPatternParser .Select(s => (CaptureContentExpression) new MatchTypeContentExpression(s.ToStringValue())); static readonly TextParser GroupedContent = - Superpower.Parse.Ref(() => Elements) + Span.EqualTo("=") + .IgnoreThen(Superpower.Parse.Ref(() => Elements)) .Select(els => (CaptureContentExpression) new GroupedContentExpression(new ExtractionPattern(els))); static readonly TextParser CaptureContent = NonGreedyContent - .Or(MatchTypeContent).Try() + .Or(MatchTypeContent) .Or(GroupedContent); static readonly TextParser Capture = diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 03cecae2..000f64ce 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -53,9 +53,9 @@ public void CaptureNameAndTypeAreParsed() [InlineData("}", false)] [InlineData("{a} b{c} ", true)] [InlineData("d {a}b {c}", true)] - [InlineData("{:{@m}}", true)] - [InlineData("Loaded {SignalId:signal-{:nat}}", true)] - [InlineData("{:{Year:num}-{Month:num}}", true)] + [InlineData("{:={@m}}", true)] + [InlineData("Loaded {SignalId:=signal-{:nat}}", true)] + [InlineData("{:={Year:num}-{Month:num}}", true)] public void OnlyValidPatternsAreAccepted(string attempt, bool isValid) { if (isValid) From e912be053dda46a4ff474735471a3bc10e511cb5 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Mon, 26 Feb 2018 20:38:36 +1000 Subject: [PATCH 20/26] Doc tweaks --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5fb4e25b..b80bbbad 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ Extraction patterns have a simple high-level syntax: * Text between `{curly braces}` is a _match expression_ that identifies a part of the event to be extracted, and * Literal curly braces are escaped by doubling, so `{{` will match the literal text `{`, and `}}` matches `}`. -Match expressions have the syntax: +Match expressions have the form: ``` {name:matcher} @@ -204,4 +204,17 @@ There are three kinds of matchers: * Matchers like `alpha` and `nat` are built-in _named_ matchers. These are built-in. * The special matchers `*`, `**` and so-on, are _non-greedy content_ matchers; these will match any text up until the next pattern element matches (`*`), the next two elements match, and so-on. We saw this in action with the `{@m:*}{:n}` elements in the example - the message is all of the text up until the next newline. * More complex _compound_ matchers are described using a sub-expression. These are prefixed with an equals sign `=`, like `{Phone:={:nat}-{:nat}-{:nat}}`. This will extract chunks of text like `123-456-7890` into the `Phone` property. - \ No newline at end of file + +### Processing + +Extraction patterns are processed from left to right. When the first non-matching pattern is encountered, extraction stops; any remaining text that couldn't be matched will be attached to the resulting event in an `@unmatched` property. + +Multi-line events are handled by looking for lines that start with the first element of the extraction pattern to be used. This works well if the first line of each event begins with something unambiguous like an `iso8601dt` timestamp; if the lines begin with less specific syntax, the first few elements of the extraction pattern might be grouped to identify the start of events more accurately: + +``` +{:=[{@t} {@l}]} {@m:*} +``` + +Here the literal text `[`, a timestamp token, adjacent space ` `, level and closing `]` are all grouped so that they constitute a single logical pattern element to identify the start of events. + +When logs are streamed into `seqcli ingest` in real time, a 10 ms deadline is applied, within which any trailing lines that make up the event must be received. From 5f48d41c3816849f410959bd6ee6e480a5107185 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sun, 25 Feb 2018 11:30:44 +1000 Subject: [PATCH 21/26] Initial work on API Key management --- src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs | 50 +++++++++++++++++++ .../Cli/Commands/ApiKey/RemoveCommand.cs | 47 +++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs create mode 100644 src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs diff --git a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs new file mode 100644 index 00000000..19e33480 --- /dev/null +++ b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json; +using SeqCli.Cli.Features; +using SeqCli.Connection; +using Serilog; + +namespace SeqCli.Cli.Commands.ApiKey +{ + [Command("apikey", "list", "Send a structured log event to the server", Example = + "seqcli log -m 'Hello, {Name}!' -p Name=World -p App=Test")] + class ListCommand : Command + { + private readonly SeqConnectionFactory _connectionFactory; + private readonly ConnectionFeature _connection; + + public ListCommand(SeqConnectionFactory connectionFactory) + { + _connectionFactory = connectionFactory; + _connection = Enable(); + } + + protected override async Task Run() + { + var connection = _connectionFactory.Connect(_connection); + + var apiKeys = await connection.ApiKeys.ListAsync(); + Log.Debug("Retrieved ApiKeys {@ApiKeys}", apiKeys); + var data = apiKeys.Select(a => new + { + a.AppliedProperties, + a.CanActAsPrincipal, + a.InputFilter, + a.Title, + a.Token, + a.UseServerTimestamps, + a.MinimumLevel, + a.IsDefault + }); + foreach (var apiKey in data) + { + var apiKeyString = JsonConvert.SerializeObject(apiKey); + + Console.WriteLine(apiKeyString); + } + return 0; + } + } +} diff --git a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs new file mode 100644 index 00000000..f3254e97 --- /dev/null +++ b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Seq.Api.Model.Inputs; +using SeqCli.Cli.Features; +using SeqCli.Connection; + +namespace SeqCli.Cli.Commands.ApiKey +{ + [Command("apikey", "remove", "Send a structured log event to the server", Example = + "seqcli log -m 'Hello, {Name}!' -p Name=World -p App=Test")] + class RemoveCommand : Command + { + private readonly SeqConnectionFactory _connectionFactory; + private readonly ConnectionFeature _connection; + private string _title; + + public RemoveCommand(SeqConnectionFactory connectionFactory) + { + _connectionFactory = connectionFactory; + _connection = Enable(); + Options.Add( + "t=|title=", + "", + (t) => _title = t); + } + + protected override async Task Run() + { + var connection = _connectionFactory.Connect(_connection); + + var apiKeys = await connection.ApiKeys.ListAsync(); + var apiKeyToRemove = apiKeys.FirstOrDefault(ak => ak.Title == _title); + if (apiKeyToRemove == null) + { + Console.WriteLine($"\"{_title}\" API Key doesn't exist"); + return -1; + } + + await connection.ApiKeys.RemoveAsync(apiKeyToRemove); + Console.WriteLine($"\"{_title}\" API Key removed"); + return 0; + } + } +} From 3e7951852e9a8a2b0c1d9c53fc5a9e5e72203a21 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Mon, 26 Feb 2018 22:58:35 +1000 Subject: [PATCH 22/26] Fixed up to be more in line with the rest of the project based on feedback --- src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs | 10 ++++--- .../Cli/Commands/ApiKey/RemoveCommand.cs | 28 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs index 19e33480..8c6a02b6 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs @@ -9,7 +9,7 @@ namespace SeqCli.Cli.Commands.ApiKey { [Command("apikey", "list", "Send a structured log event to the server", Example = - "seqcli log -m 'Hello, {Name}!' -p Name=World -p App=Test")] + "seqcli apikey list")] class ListCommand : Command { private readonly SeqConnectionFactory _connectionFactory; @@ -29,15 +29,17 @@ protected override async Task Run() Log.Debug("Retrieved ApiKeys {@ApiKeys}", apiKeys); var data = apiKeys.Select(a => new { + a.Title, + a.Id, + a.Token, + a.MinimumLevel, a.AppliedProperties, a.CanActAsPrincipal, a.InputFilter, - a.Title, - a.Token, a.UseServerTimestamps, - a.MinimumLevel, a.IsDefault }); + foreach (var apiKey in data) { var apiKeyString = JsonConvert.SerializeObject(apiKey); diff --git a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs index f3254e97..84b43b2f 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs @@ -9,13 +9,14 @@ namespace SeqCli.Cli.Commands.ApiKey { - [Command("apikey", "remove", "Send a structured log event to the server", Example = - "seqcli log -m 'Hello, {Name}!' -p Name=World -p App=Test")] + [Command("apikey", "remove", "Remove API Key from the server", Example = + "seqcli apikey remove -t TestApiKey")] class RemoveCommand : Command { private readonly SeqConnectionFactory _connectionFactory; private readonly ConnectionFeature _connection; private string _title; + private string _id; public RemoveCommand(SeqConnectionFactory connectionFactory) { @@ -23,24 +24,37 @@ public RemoveCommand(SeqConnectionFactory connectionFactory) _connection = Enable(); Options.Add( "t=|title=", - "", + "Remove API Keys with the specified title", (t) => _title = t); + + Options.Add( + "i=|id=", + "Remove API Keys with the specified Id", + (t) => _id = t); } protected override async Task Run() { + if (_title != default && _id != default) + { + Console.WriteLine("You can only specify \"title\" or \"id\" not both"); + return -1; + } + var connection = _connectionFactory.Connect(_connection); var apiKeys = await connection.ApiKeys.ListAsync(); - var apiKeyToRemove = apiKeys.FirstOrDefault(ak => ak.Title == _title); - if (apiKeyToRemove == null) + var apiKeyToRemove = apiKeys.Where(ak => ak.Title == _title || ak.Id == _id).ToList(); + if (!apiKeyToRemove.Any()) { Console.WriteLine($"\"{_title}\" API Key doesn't exist"); return -1; } - await connection.ApiKeys.RemoveAsync(apiKeyToRemove); - Console.WriteLine($"\"{_title}\" API Key removed"); + foreach (var apiKeyEntity in apiKeyToRemove) + { + await connection.ApiKeys.RemoveAsync(apiKeyEntity); + } return 0; } } From 84012f49e07b791dd02cd19e783eb2bc87a93666 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Wed, 28 Feb 2018 06:30:43 +1000 Subject: [PATCH 23/26] Fix up help text for apikey list command --- src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs index 8c6a02b6..393c9bb7 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs @@ -8,7 +8,7 @@ namespace SeqCli.Cli.Commands.ApiKey { - [Command("apikey", "list", "Send a structured log event to the server", Example = + [Command("apikey", "list", "List of API Keys", Example = "seqcli apikey list")] class ListCommand : Command { From e25bfc1dc2f6005913e21bb436c5f06a69c9eed6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 1 Mar 2018 20:43:35 +1000 Subject: [PATCH 24/26] Generic entity formatting spike --- README.md | 4 +- src/SeqCli/Cli/CommandLineHost.cs | 9 ++- src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs | 38 ++++------ .../Cli/Commands/ApiKey/RemoveCommand.cs | 12 ++-- src/SeqCli/Cli/Commands/HelpCommand.cs | 71 ++++++++++++++----- .../Cli/Features/OutputFormatFeature.cs | 39 ++++++++++ .../Output/StripStructureTypeEnricher.cs | 25 +++++++ src/SeqCli/SeqCli.csproj | 1 + 8 files changed, 145 insertions(+), 54 deletions(-) create mode 100644 src/SeqCli/Output/StripStructureTypeEnricher.cs diff --git a/README.md b/README.md index 00b62546..f84ae153 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Stream log events matching a filter. Print the current executable version. -## Extraction Patterns +## Extraction patterns The `seqcli ingest` command can be used for parsing plain text logs into structured log events. @@ -201,7 +201,7 @@ Different matchers are needed so that a piece of text like `200OK` can be separa There are three kinds of matchers: - * Matchers like `alpha` and `nat` are built-in _named_ matchers. These are built-in. + * Matchers like `alpha` and `nat` are built-in _named_ matchers. * The special matchers `*`, `**` and so-on, are _non-greedy content_ matchers; these will match any text up until the next pattern element matches (`*`), the next two elements match, and so-on. We saw this in action with the `{@m:*}{:n}` elements in the example - the message is all of the text up until the next newline. * More complex _compound_ matchers are described using a sub-expression. These are prefixed with an equals sign `=`, like `{Phone:={:nat}-{:nat}-{:nat}}`. This will extract chunks of text like `123-456-7890` into the `Phone` property. diff --git a/src/SeqCli/Cli/CommandLineHost.cs b/src/SeqCli/Cli/CommandLineHost.cs index bc725e3c..bf78fb1d 100644 --- a/src/SeqCli/Cli/CommandLineHost.cs +++ b/src/SeqCli/Cli/CommandLineHost.cs @@ -38,11 +38,14 @@ public async Task Run(string[] args) if (args.Length > 0) { var norm = args[0].ToLowerInvariant(); - var subCommandNorm = args.Length > 1 && !args[1].Contains("-") ? args[1].ToLowerInvariant() : default; - var cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == norm && c.Metadata.SubCommand == subCommandNorm); + var subCommandNorm = args.Length > 1 && !args[1].Contains("-") ? args[1].ToLowerInvariant() : null; + + var cmd = _availableCommands.SingleOrDefault(c => + c.Metadata.Name == norm && (c.Metadata.SubCommand == subCommandNorm || c.Metadata.SubCommand == null)); + if (cmd != null) { - var amountToSkip = subCommandNorm == default ? 1 : 2; + var amountToSkip = cmd.Metadata.SubCommand == null ? 1 : 2; return await cmd.Value.Value.Invoke(args.Skip(amountToSkip).ToArray()); } } diff --git a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs index 393c9bb7..432f75a5 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/ListCommand.cs @@ -1,23 +1,26 @@ using System; -using System.Linq; using System.Threading.Tasks; -using Newtonsoft.Json; using SeqCli.Cli.Features; +using SeqCli.Config; using SeqCli.Connection; using Serilog; namespace SeqCli.Cli.Commands.ApiKey { - [Command("apikey", "list", "List of API Keys", Example = - "seqcli apikey list")] + [Command("apikey", "list", "List API keys on the server", Example="seqcli apikey list")] class ListCommand : Command { - private readonly SeqConnectionFactory _connectionFactory; - private readonly ConnectionFeature _connection; + readonly SeqConnectionFactory _connectionFactory; + + readonly ConnectionFeature _connection; + readonly OutputFormatFeature _output; - public ListCommand(SeqConnectionFactory connectionFactory) + public ListCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config) { - _connectionFactory = connectionFactory; + if (config == null) throw new ArgumentNullException(nameof(config)); + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + + _output = Enable(new OutputFormatFeature(config.Output)); _connection = Enable(); } @@ -27,25 +30,12 @@ protected override async Task Run() var apiKeys = await connection.ApiKeys.ListAsync(); Log.Debug("Retrieved ApiKeys {@ApiKeys}", apiKeys); - var data = apiKeys.Select(a => new - { - a.Title, - a.Id, - a.Token, - a.MinimumLevel, - a.AppliedProperties, - a.CanActAsPrincipal, - a.InputFilter, - a.UseServerTimestamps, - a.IsDefault - }); - foreach (var apiKey in data) + foreach (var apiKey in apiKeys) { - var apiKeyString = JsonConvert.SerializeObject(apiKey); - - Console.WriteLine(apiKeyString); + _output.WriteEntity(apiKey); } + return 0; } } diff --git a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs index 84b43b2f..98fb39e7 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/RemoveCommand.cs @@ -9,14 +9,14 @@ namespace SeqCli.Cli.Commands.ApiKey { - [Command("apikey", "remove", "Remove API Key from the server", Example = - "seqcli apikey remove -t TestApiKey")] + [Command("apikey", "remove", "Remove an API key from the server", + Example="seqcli apikey remove -t TestApiKey")] class RemoveCommand : Command { - private readonly SeqConnectionFactory _connectionFactory; - private readonly ConnectionFeature _connection; - private string _title; - private string _id; + readonly SeqConnectionFactory _connectionFactory; + readonly ConnectionFeature _connection; + string _title; + string _id; public RemoveCommand(SeqConnectionFactory connectionFactory) { diff --git a/src/SeqCli/Cli/Commands/HelpCommand.cs b/src/SeqCli/Cli/Commands/HelpCommand.cs index dda56ae8..13161ada 100644 --- a/src/SeqCli/Cli/Commands/HelpCommand.cs +++ b/src/SeqCli/Cli/Commands/HelpCommand.cs @@ -37,13 +37,29 @@ protected override Task Run(string[] unrecognised) { var ea = Assembly.GetEntryAssembly(); var name = ea.GetName().Name; - + + if (_markdown) + { + if (unrecognised.Length != 0) + return base.Run(unrecognised); + + PrintMarkdownHelp(name); + return Task.FromResult(0); + } + + string topLevelCommand = null; if (unrecognised.Length > 0) { - var target = unrecognised[0].ToLowerInvariant(); - var cmd = _availableCommands.SingleOrDefault(c => c.Metadata.Name == target); - if (cmd != null) + topLevelCommand = unrecognised[0].ToLowerInvariant(); + var subCommand = unrecognised.Length > 1 && !unrecognised[1].Contains("-") ? unrecognised[1] : null; + var cmds = _availableCommands.Where(c => c.Metadata.Name == topLevelCommand && + (subCommand == null || subCommand == c.Metadata.SubCommand)).ToArray(); + if (cmds.Length == 0) + return base.Run(unrecognised); + + if (cmds.Length == 1) { + var cmd = cmds.Single(); var argHelp = cmd.Value.Value.HasArgs ? " []" : ""; Console.WriteLine(name + " " + cmd.Metadata.Name + argHelp); Console.WriteLine(); @@ -52,19 +68,13 @@ protected override Task Run(string[] unrecognised) cmd.Value.Value.PrintUsage(); return Task.FromResult(0); } - - return base.Run(unrecognised); } - if (_markdown) - { - PrintMarkdownHelp(name); - } + if (topLevelCommand != null) + PrintHelp(name, topLevelCommand); else - { PrintHelp(name); - } - + return Task.FromResult(0); } @@ -126,17 +136,40 @@ void PrintHelp(string executableName) Console.WriteLine(); Console.WriteLine("Available commands are:"); - foreach (var availableCommand in _availableCommands) + var printedGroups = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var avail in _availableCommands) { - Printing.Define( - " " + availableCommand.Metadata.Name, - availableCommand.Metadata.HelpText, - 13, - Console.Out); + if (avail.Metadata.SubCommand != null) + { + if (!printedGroups.Contains(avail.Metadata.Name)) + { + Printing.Define($" {avail.Metadata.Name}", "", 13, Console.Out); + printedGroups.Add(avail.Metadata.Name); + } + } + else + { + Printing.Define($" {avail.Metadata.Name}", avail.Metadata.HelpText, 13, Console.Out); + } } Console.WriteLine(); Console.WriteLine($"Type `{executableName} help ` for detailed help"); } + + void PrintHelp(string executableName, string topLevelCommand) + { + Console.WriteLine($"Usage: {executableName} {topLevelCommand} []"); + Console.WriteLine(); + Console.WriteLine("Available sub-commands are:"); + + foreach (var avail in _availableCommands.Where(c => c.Metadata.Name == topLevelCommand)) + { + Printing.Define($" {avail.Metadata.SubCommand}", avail.Metadata.HelpText, 13, Console.Out); + } + + Console.WriteLine(); + Console.WriteLine($"Type `{executableName} help {topLevelCommand} ` for detailed help"); + } } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index 506b231b..535e6502 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -13,6 +13,10 @@ // limitations under the License. using System; +using Destructurama; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Seq.Api.Model; using SeqCli.Config; using SeqCli.Csv; using SeqCli.Output; @@ -75,5 +79,40 @@ public void WriteCsv(string csv) CsvWriter.WriteCsv(tokens, Theme, Console.Out, true); } } + + public void WriteEntity(Entity entity) + { + if (entity == null) throw new ArgumentNullException(nameof(entity)); + + var jo = JObject.FromObject(entity); + + if (_json) + { + jo.Remove("Links"); + if (_noColor) + { + Console.WriteLine(JsonConvert.SerializeObject(jo, Formatting.None)); + } + else + { + // Proof-of-concept; this is a very inefficient + // way to write colorized JSON ;) + + var writer = new LoggerConfiguration() + .Destructure.JsonNetTypes() + .Enrich.With() + .WriteTo.Console( + outputTemplate: "{@Message:j}{NewLine}", + theme: Theme) + .CreateLogger(); + writer.Information("{@Entity}", jo); + } + } + else + { + var dyn = (dynamic) jo; + Console.WriteLine($"{entity.Id} {dyn.Title ?? dyn.Name}"); + } + } } } diff --git a/src/SeqCli/Output/StripStructureTypeEnricher.cs b/src/SeqCli/Output/StripStructureTypeEnricher.cs new file mode 100644 index 00000000..b50b1831 --- /dev/null +++ b/src/SeqCli/Output/StripStructureTypeEnricher.cs @@ -0,0 +1,25 @@ +using System.Linq; +using Serilog.Core; +using Serilog.Data; +using Serilog.Events; + +namespace SeqCli.Output +{ + public class StripStructureTypeEnricher : LogEventPropertyValueRewriter, ILogEventEnricher + { + public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + { + foreach (var property in logEvent.Properties) + { + var updated = new LogEventProperty(property.Key, Visit(null, property.Value)); + logEvent.AddOrUpdateProperty(updated); + } + } + + protected override LogEventPropertyValue VisitStructureValue(object state, StructureValue structure) + { + return new StructureValue(structure.Properties.Select(p => + new LogEventProperty(p.Name, Visit(null, p.Value)))); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 0292c67e..75f4ec66 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -18,6 +18,7 @@ + From 5c520a0efd5697c8f70a7a9fc942d6453f7d703b Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 1 Mar 2018 20:47:31 +1000 Subject: [PATCH 25/26] Might as well line up colorized and non-colorized output --- .../Cli/Features/OutputFormatFeature.cs | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index 535e6502..6cfbd0a7 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -89,24 +89,17 @@ public void WriteEntity(Entity entity) if (_json) { jo.Remove("Links"); - if (_noColor) - { - Console.WriteLine(JsonConvert.SerializeObject(jo, Formatting.None)); - } - else - { - // Proof-of-concept; this is a very inefficient - // way to write colorized JSON ;) - - var writer = new LoggerConfiguration() - .Destructure.JsonNetTypes() - .Enrich.With() - .WriteTo.Console( - outputTemplate: "{@Message:j}{NewLine}", - theme: Theme) - .CreateLogger(); - writer.Information("{@Entity}", jo); - } + // Proof-of-concept; this is a very inefficient + // way to write colorized JSON ;) + + var writer = new LoggerConfiguration() + .Destructure.JsonNetTypes() + .Enrich.With() + .WriteTo.Console( + outputTemplate: "{@Message:j}{NewLine}", + theme: Theme) + .CreateLogger(); + writer.Information("{@Entity}", jo); } else { From 3353661536095a9ba53d23d591861f6ef75b63a8 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 2 Mar 2018 08:41:13 +1000 Subject: [PATCH 26/26] Drop test cases for (currently) unsupported optional-subcommand scenario --- test/SeqCli.Tests/Cli/CommandLineHostTests.cs | 51 ++----------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs index 049c24df..732bcd7a 100644 --- a/test/SeqCli.Tests/Cli/CommandLineHostTests.cs +++ b/test/SeqCli.Tests/Cli/CommandLineHostTests.cs @@ -1,12 +1,9 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; using Autofac.Features.Metadata; using SeqCli.Cli; -using SeqCli.Cli.Commands; using Xunit; namespace SeqCli.Tests.Cli @@ -21,10 +18,10 @@ public async Task CheckCommandLineHostPicksCorrectCommand() { new Meta, CommandMetadata>( new Lazy(() => new ActionCommand(() => commandsRan.Add("test"))), - new CommandMetadata() {Name = "test"}), + new CommandMetadata {Name = "test"}), new Meta, CommandMetadata>( new Lazy(() => new ActionCommand(() => commandsRan.Add("test2"))), - new CommandMetadata() {Name = "test2"}) + new CommandMetadata {Name = "test2"}) }; var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(new []{ "test"}); @@ -32,46 +29,6 @@ public async Task CheckCommandLineHostPicksCorrectCommand() Assert.Equal(commandsRan.First(), "test"); } - [Fact] - public async Task WhenCommandAndSubcommandAndTheUserRunsWithoutSubcommandEnsurePickedCorrect() - { - var commandsRan = new List(); - var availableCommands = - new List, CommandMetadata>> - { - new Meta, CommandMetadata>( - new Lazy(() => new ActionCommand(() => commandsRan.Add("test"))), - new CommandMetadata() {Name = "test"}), - new Meta, CommandMetadata>( - new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand"))), - new CommandMetadata() {Name = "test", SubCommand = "subcommand"}) - }; - var commandLineHost = new CommandLineHost(availableCommands); - await commandLineHost.Run(new[] { "test" }); - - Assert.Equal(commandsRan.First(), "test"); - } - - [Fact] - public async Task WhenCommandAndSubcommandAndTheUserRunsWithSubcommandEnsurePickedCorrect() - { - var commandsRan = new List(); - var availableCommands = - new List, CommandMetadata>> - { - new Meta, CommandMetadata>( - new Lazy(() => new ActionCommand(() => commandsRan.Add("test"))), - new CommandMetadata() {Name = "test"}), - new Meta, CommandMetadata>( - new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand"))), - new CommandMetadata() {Name = "test", SubCommand = "subcommand"}) - }; - var commandLineHost = new CommandLineHost(availableCommands); - await commandLineHost.Run(new[] { "test", "subcommand" }); - - Assert.Equal(commandsRan.First(), "test-subcommand"); - } - [Fact] public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePickedCorrect() { @@ -81,10 +38,10 @@ public async Task WhenMoreThanOneSubcommandAndTheUserRunsWithSubcommandEnsurePic { new Meta, CommandMetadata>( new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand1"))), - new CommandMetadata() {Name = "test", SubCommand = "subcommand1"}), + new CommandMetadata {Name = "test", SubCommand = "subcommand1"}), new Meta, CommandMetadata>( new Lazy(() => new ActionCommand(() => commandsRan.Add("test-subcommand2"))), - new CommandMetadata() {Name = "test", SubCommand = "subcommand2"}) + new CommandMetadata {Name = "test", SubCommand = "subcommand2"}) }; var commandLineHost = new CommandLineHost(availableCommands); await commandLineHost.Run(new[] { "test", "subcommand2" });