Improve caching: found tokens in token scanner, resolved resources and XObject forms

src/UglyToad.PdfPig.Tests/ContentTests/ResourceStoreCachingTests.cs

@@ -0,0 +1,104 @@
+namespace UglyToad.PdfPig.Tests.ContentTests
+{
+    using System.Collections.Generic;
+    using PdfPig.Content;
+    using PdfPig.Core;
+    using PdfPig.PdfFonts;
+    using PdfPig.Tokens;
+    using PdfPig.Tests.Tokens;
+    using Xunit;
+
+    /// <summary>
+    /// Issue #1390: a page with thousands of form XObjects sharing one resource dictionary re-expanded that
+    /// dictionary on every invocation, which is quadratic in the number of resource entries.
+    /// </summary>
+    public class ResourceStoreCachingTests
+    {
+        private sealed class NoOpFontFactory : IFontFactory
+        {
+            public IFont Get(DictionaryToken dictionary) => null!;
+        }
+
+        private static ResourceStore BuildStore(TestPdfTokenScanner scanner)
+        {
+            return new ResourceStore(
+                scanner,
+                new NoOpFontFactory(),
+                new TestFilterProvider(),
+                new ParsingOptions
+                {
+                    UseLenientParsing = true,
+                    SkipMissingFonts = true,
+                });
+        }
+
+        /// <summary>
+        /// Builds `&lt;&lt; /ExtGState 20 0 R &gt;&gt;` where object 20 is `&lt;&lt; /G0 21 0 R /G1 22 0 R &gt;&gt;`,
+        /// matching the shape of the document in issue #1390.
+        /// </summary>
+        private static DictionaryToken RegisterResourcesWithIndirectExtGState(TestPdfTokenScanner scanner)
+        {
+            var extGStateReference = new IndirectReference(20, 0);
+            var g0Reference = new IndirectReference(21, 0);
+            var g1Reference = new IndirectReference(22, 0);
+
+            void Register(IndirectReference reference, IToken token)
+                => scanner.Objects[reference] = new ObjectToken(XrefLocation.File(0), reference, token);
+
+            Register(g0Reference, new DictionaryToken(new Dictionary<NameToken, IToken>
+            {
+                { NameToken.Lw, new NumericToken(1) }
+            }));
+
+            Register(g1Reference, new DictionaryToken(new Dictionary<NameToken, IToken>
+            {
+                { NameToken.Lw, new NumericToken(2) }
+            }));
+
+            Register(extGStateReference, new DictionaryToken(new Dictionary<NameToken, IToken>
+            {
+                { NameToken.Create("G0"), new IndirectReferenceToken(g0Reference) },
+                { NameToken.Create("G1"), new IndirectReferenceToken(g1Reference) }
+            }));
+
+            return new DictionaryToken(new Dictionary<NameToken, IToken>
+            {
+                { NameToken.ExtGState, new IndirectReferenceToken(extGStateReference) }
+            });
+        }
+
+        [Fact]
+        public void ReloadingTheSameResourceDictionaryResolvesNoFurtherObjects()
+        {
+            var scanner = new TestPdfTokenScanner();
+            var resources = RegisterResourcesWithIndirectExtGState(scanner);
+            var store = BuildStore(scanner);
+
+            store.LoadResourceDictionary(resources);
+            store.UnloadResourceDictionary();
+
+            var afterFirstLoad = scanner.GetCallCount;
+
+            store.LoadResourceDictionary(resources);
+
+            Assert.Equal(afterFirstLoad, scanner.GetCallCount);
+        }
+
+        [Fact]
+        public void ReloadingTheSameResourceDictionaryStillResolvesItsEntries()
+        {
+            var scanner = new TestPdfTokenScanner();
+            var resources = RegisterResourcesWithIndirectExtGState(scanner);
+            var store = BuildStore(scanner);
+
+            store.LoadResourceDictionary(resources);
+            var firstLoad = store.GetExtendedGraphicsStateDictionary(NameToken.Create("G1"));
+            store.UnloadResourceDictionary();
+
+            store.LoadResourceDictionary(resources);
+            var secondLoad = store.GetExtendedGraphicsStateDictionary(NameToken.Create("G1"));
+
+            Assert.Same(firstLoad, secondLoad);
+        }
+    }
+}

src/UglyToad.PdfPig.Tests/Graphics/FormXObjectCachingTests.cs

@@ -0,0 +1,148 @@
+namespace UglyToad.PdfPig.Tests.Graphics
+{
+    using System.Collections.Generic;
+    using PdfPig.Content;
+    using PdfPig.Core;
+    using PdfPig.Geometry;
+    using PdfPig.Graphics;
+    using PdfPig.Graphics.Operations;
+    using PdfPig.Logging;
+    using PdfPig.Parser;
+    using PdfPig.PdfFonts;
+    using PdfPig.Tokens;
+    using PdfPig.Tests.Tokens;
+    using Xunit;
+
+    /// <summary>
+    /// Issue #1390: a page can invoke the same form XObject thousands of times. Every invocation used to
+    /// resolve the form's stream again and re-parse its content stream.
+    /// </summary>
+    public class FormXObjectCachingTests
+    {
+        private static readonly NameToken FormName = NameToken.Create("Fm0");
+
+        private static readonly IndirectReference FormReference = new IndirectReference(5, 0);
+
+        private sealed class NoOpFontFactory : IFontFactory
+        {
+            public IFont Get(DictionaryToken dictionary) => null!;
+        }
+
+        private sealed class CountingPageContentParser : IPageContentParser
+        {
+            private readonly IPageContentParser inner;
+
+            public CountingPageContentParser(IPageContentParser inner) => this.inner = inner;
+
+            public int ParseCallCount { get; private set; }
+
+            public IReadOnlyList<IGraphicsStateOperation> Parse(int pageNumber, IInputBytes inputBytes, ILog log)
+            {
+                ParseCallCount++;
+
+                return inner.Parse(pageNumber, inputBytes, log);
+            }
+        }
+
+        private static TestPdfTokenScanner CreateScannerWithForm()
+        {
+            var scanner = new TestPdfTokenScanner();
+
+            var formDictionary = new DictionaryToken(new Dictionary<NameToken, IToken>
+            {
+                { NameToken.Type, NameToken.Xobject },
+                { NameToken.Subtype, NameToken.Form },
+                {
+                    NameToken.Bbox, new ArrayToken(new IToken[]
+                    {
+                        new NumericToken(0), new NumericToken(0), new NumericToken(10), new NumericToken(10)
+                    })
+                }
+            });
+
+            var formStream = new StreamToken(formDictionary, OtherEncodings.StringAsLatin1Bytes("0 0 10 10 re f\n"));
+
+            scanner.Objects[FormReference] = new ObjectToken(XrefLocation.File(0), FormReference, formStream);
+
+            return scanner;
+        }
+
+        private static ContentStreamProcessor CreateProcessor(TestPdfTokenScanner scanner,
+            IPageContentParser pageContentParser)
+        {
+            var parsingOptions = new ParsingOptions { UseLenientParsing = true, SkipMissingFonts = true };
+
+            var resourceStore = new ResourceStore(scanner, new NoOpFontFactory(), new TestFilterProvider(), parsingOptions);
+
+            resourceStore.LoadResourceDictionary(new DictionaryToken(new Dictionary<NameToken, IToken>
+            {
+                {
+                    NameToken.Xobject, new DictionaryToken(new Dictionary<NameToken, IToken>
+                    {
+                        { FormName, new IndirectReferenceToken(FormReference) }
+                    })
+                }
+            }));
+
+            return new ContentStreamProcessor(
+                1,
+                resourceStore,
+                scanner,
+                pageContentParser,
+                new TestFilterProvider(),
+                new CropBox(new PdfRectangle(0, 0, 612, 792)),
+                UserSpaceUnit.Default,
+                new PageRotationDegrees(0),
+                TransformationMatrix.Identity,
+                parsingOptions);
+        }
+
+        private static CountingPageContentParser CreateParser()
+        {
+            return new CountingPageContentParser(
+                new PageContentParser(ReflectionGraphicsStateOperationFactory.Instance, new StackDepthGuard(256)));
+        }
+
+        [Fact]
+        public void RepeatedFormInvocationParsesTheContentStreamOnce()
+        {
+            var scanner = CreateScannerWithForm();
+            var parser = CreateParser();
+            var processor = CreateProcessor(scanner, parser);
+
+            processor.ApplyXObject(FormName);
+            processor.ApplyXObject(FormName);
+
+            Assert.Equal(1, parser.ParseCallCount);
+        }
+
+        [Fact]
+        public void RepeatedFormInvocationResolvesTheStreamOnce()
+        {
+            var scanner = CreateScannerWithForm();
+            var processor = CreateProcessor(scanner, CreateParser());
+
+            processor.ApplyXObject(FormName);
+            var afterFirstInvocation = scanner.GetCallCount;
+
+            processor.ApplyXObject(FormName);
+
+            Assert.Equal(afterFirstInvocation, scanner.GetCallCount);
+        }
+
+        [Fact]
+        public void RepeatedFormInvocationRunsTheContentEveryTime()
+        {
+            var scanner = CreateScannerWithForm();
+            var processor = CreateProcessor(scanner, CreateParser());
+
+            processor.ApplyXObject(FormName);
+            processor.ApplyXObject(FormName);
+            processor.ApplyXObject(FormName);
+
+            var content = processor.Process(1, new List<IGraphicsStateOperation>());
+
+            Assert.Equal(3, content.Paths.Count);
+        }
+    }
+}

src/UglyToad.PdfPig.Tests/Tokenization/Scanner/PdfTokenScannerTests.cs

@@ -717,6 +717,39 @@ 2 0 obj
             Assert.IsType<NumericToken>(lengthValue);
         }
 
+        [Fact]
+        public void GetResolvesObjectFromXrefTableOnlyOnce()
+        {
+            // Issue #1390: objects located via a classic xref table were re-read and re-tokenized
+            // on every lookup because only the object stream and brute force paths populated the cache.
+            const string s = "12 0 obj\n<< /Type /Page >>\nendobj\n";
+
+            var reference = new IndirectReference(12, 0);
+            var scanner = GetScannerWithRealLocationProvider(s, (reference, 0));
+
+            var first = scanner.Get(reference);
+            var second = scanner.Get(reference);
+
+            Assert.Same(first, second);
+        }
+
+        [Fact]
+        public void GetDoesNotCacheStreamObjects()
+        {
+            // Streams are deliberately left out of the object cache. Caching them would pin the raw
+            // bytes of every image and content stream ever resolved for the lifetime of the document.
+            const string s = "7 0 obj\n<< /Length 11 >>\nstream\nhello world\nendstream\nendobj\n";
+
+            var reference = new IndirectReference(7, 0);
+            var scanner = GetScannerWithRealLocationProvider(s, (reference, 0));
+
+            var first = scanner.Get(reference);
+            var second = scanner.Get(reference);
+
+            Assert.IsType<StreamToken>(first.Data);
+            Assert.NotSame(first, second);
+        }
+
         private static PdfTokenScanner GetScanner(string s, TestObjectLocationProvider locationProvider = null, bool useLenientParsing = false)
         {
             var input = StringBytesTestConverter.Convert(s, false);
@@ -730,6 +763,23 @@ private static PdfTokenScanner GetScanner(string s, TestObjectLocationProvider l
                 new StackDepthGuard(256));
         }
 
+
+        private static PdfTokenScanner GetScannerWithRealLocationProvider(string s,
+            params (IndirectReference Reference, long Offset)[] offsets)
+        {
+            var input = StringBytesTestConverter.Convert(s, false);
+
+            var xrefOffsets = offsets.ToDictionary(x => x.Reference, x => XrefLocation.File(x.Offset));
+
+            return new PdfTokenScanner(input.Bytes,
+                new ObjectLocationProvider(xrefOffsets, null, input.Bytes),
+                new TestFilterProvider(),
+                NoOpEncryptionHandler.Instance,
+                new FileHeaderOffset(0),
+                ParsingOptions.LenientParsingOff,
+                new StackDepthGuard(256));
+        }
+
         private static IReadOnlyList<ObjectToken> ReadToEnd(PdfTokenScanner scanner)
         {
             var result = new List<ObjectToken>();

src/UglyToad.PdfPig.Tests/Tokens/TestPdfTokenScanner.cs

@@ -11,7 +11,12 @@ internal class TestPdfTokenScanner : IPdfTokenScanner
         public StackDepthGuard StackDepthGuard => StackDepthGuard.Infinite;
 
         public Dictionary<IndirectReference, ObjectToken> Objects { get; } = new Dictionary<IndirectReference, ObjectToken>();
-
+
+        /// <summary>
+        /// Number of times <see cref="Get"/> has been called, so tests can assert that work is not repeated.
+        /// </summary>
+        public int GetCallCount { get; private set; }
+
         public bool MoveNext()
         {
             throw new NotImplementedException();
@@ -43,6 +48,7 @@ public void DeregisterCustomTokenizer(ITokenizer tokenizer)
 
         public ObjectToken Get(IndirectReference reference)
         {
+            GetCallCount++;
             return Objects[reference];
         }