Skip to content

Commit ad51267

Browse files
authored
Add write-raw APIs to Utf8JsonWriter (#54254)
* Add write-raw APIs to Utf8JsonWriter * Address review feedback * Address review feedback * Address review feedback * Make long-running tests outerloop * Fix call to core logic
1 parent ebba1d4 commit ad51267

8 files changed

Lines changed: 770 additions & 8 deletions

File tree

src/libraries/System.Text.Json/ref/System.Text.Json.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,9 @@ public void WritePropertyName(System.ReadOnlySpan<byte> utf8PropertyName) { }
473473
public void WritePropertyName(System.ReadOnlySpan<char> propertyName) { }
474474
public void WritePropertyName(string propertyName) { }
475475
public void WritePropertyName(System.Text.Json.JsonEncodedText propertyName) { }
476+
public void WriteRawValue(string json, bool skipInputValidation = false) { }
477+
public void WriteRawValue(System.ReadOnlySpan<byte> utf8Json, bool skipInputValidation = false) { }
478+
public void WriteRawValue(System.ReadOnlySpan<char> json, bool skipInputValidation = false) { }
476479
public void WriteStartArray() { }
477480
public void WriteStartArray(System.ReadOnlySpan<byte> utf8PropertyName) { }
478481
public void WriteStartArray(System.ReadOnlySpan<char> propertyName) { }

src/libraries/System.Text.Json/src/System.Text.Json.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@
257257
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.Guid.cs" />
258258
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.Helpers.cs" />
259259
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.Literal.cs" />
260+
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.Raw.cs" />
260261
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.SignedNumber.cs" />
261262
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.String.cs" />
262263
<Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.UnsignedNumber.cs" />

src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,13 @@ internal static class JsonConstants
6464
// All other UTF-16 characters can be represented by either 1 or 2 UTF-8 bytes.
6565
public const int MaxExpansionFactorWhileTranscoding = 3;
6666

67+
// When transcoding from UTF8 -> UTF16, the byte count threshold where we rent from the array pool before performing a normal alloc.
68+
public const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024;
69+
70+
// The maximum number of characters allowed when writing raw UTF-16 JSON. This is the maximum length that we can guarantee can
71+
// be safely transcoded to UTF-8 and fit within an integer-length span, given the max expansion factor of a single character (3).
72+
public const int MaxUtf16RawValueLength = int.MaxValue / MaxExpansionFactorWhileTranscoding;
73+
6774
public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value.
6875
public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes
6976
public const int MaxBase64ValueTokenSize = (MaxEscapedTokenSize >> 2) * 3 / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -355,12 +355,10 @@ public static partial class JsonSerializer
355355

356356
private static TValue? ReadUsingMetadata<TValue>(ReadOnlySpan<char> json, JsonTypeInfo jsonTypeInfo)
357357
{
358-
const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024;
359-
360358
byte[]? tempArray = null;
361359

362360
// For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold.
363-
Span<byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ?
361+
Span<byte> utf8 = json.Length <= (JsonConstants.ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ?
364362
// Use a pooled alloc.
365363
tempArray = ArrayPool<byte>.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) :
366364
// Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation)
Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Buffers;
5+
using System.Diagnostics;
6+
7+
namespace System.Text.Json
8+
{
9+
public sealed partial class Utf8JsonWriter
10+
{
11+
/// <summary>
12+
/// Writes the input as JSON content. It is expected that the input content is a single complete JSON value.
13+
/// </summary>
14+
/// <param name="json">The raw JSON content to write.</param>
15+
/// <param name="skipInputValidation">Whether to validate if the input is an RFC 8259-compliant JSON payload.</param>
16+
/// <exception cref="ArgumentNullException">Thrown if <paramref name="json"/> is <see langword="null"/>.</exception>
17+
/// <exception cref="ArgumentException">Thrown if the length of the input is zero or greater than 715,827,882 (<see cref="int.MaxValue"/> / 3).</exception>
18+
/// <exception cref="JsonException">
19+
/// Thrown if <paramref name="skipInputValidation"/> is <see langword="false"/>, and the input
20+
/// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259)
21+
/// or the input JSON exceeds a recursive depth of 64.
22+
/// </exception>
23+
/// <remarks>
24+
/// When writing untrused JSON values, do not set <paramref name="skipInputValidation"/> to <see langword="true"/> as this can result in invalid JSON
25+
/// being written, and/or the overall payload being written to the writer instance being invalid.
26+
///
27+
/// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled).
28+
///
29+
/// The <see cref="JsonWriterOptions.SkipValidation"/> value for the writer instance is honored when using this method.
30+
///
31+
/// The <see cref="JsonWriterOptions.Indented"/> and <see cref="JsonWriterOptions.Encoder"/> values for the writer instance are not applied when using this method.
32+
/// </remarks>
33+
public void WriteRawValue(string json, bool skipInputValidation = false)
34+
{
35+
if (!_options.SkipValidation)
36+
{
37+
ValidateWritingValue();
38+
}
39+
40+
if (json == null)
41+
{
42+
throw new ArgumentNullException(nameof(json));
43+
}
44+
45+
TranscodeAndWriteRawValue(json.AsSpan(), skipInputValidation);
46+
}
47+
48+
/// <summary>
49+
/// Writes the input as JSON content. It is expected that the input content is a single complete JSON value.
50+
/// </summary>
51+
/// <param name="json">The raw JSON content to write.</param>
52+
/// <param name="skipInputValidation">Whether to validate if the input is an RFC 8259-compliant JSON payload.</param>
53+
/// <exception cref="ArgumentException">Thrown if the length of the input is zero or greater than 715,827,882 (<see cref="int.MaxValue"/> / 3).</exception>
54+
/// <exception cref="JsonException">
55+
/// Thrown if <paramref name="skipInputValidation"/> is <see langword="false"/>, and the input
56+
/// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259)
57+
/// or the input JSON exceeds a recursive depth of 64.
58+
/// </exception>
59+
/// <remarks>
60+
/// When writing untrused JSON values, do not set <paramref name="skipInputValidation"/> to <see langword="true"/> as this can result in invalid JSON
61+
/// being written, and/or the overall payload being written to the writer instance being invalid.
62+
///
63+
/// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled).
64+
///
65+
/// The <see cref="JsonWriterOptions.SkipValidation"/> value for the writer instance is honored when using this method.
66+
///
67+
/// The <see cref="JsonWriterOptions.Indented"/> and <see cref="JsonWriterOptions.Encoder"/> values for the writer instance are not applied when using this method.
68+
/// </remarks>
69+
public void WriteRawValue(ReadOnlySpan<char> json, bool skipInputValidation = false)
70+
{
71+
if (!_options.SkipValidation)
72+
{
73+
ValidateWritingValue();
74+
}
75+
76+
TranscodeAndWriteRawValue(json, skipInputValidation);
77+
}
78+
79+
/// <summary>
80+
/// Writes the input as JSON content. It is expected that the input content is a single complete JSON value.
81+
/// </summary>
82+
/// <param name="utf8Json">The raw JSON content to write.</param>
83+
/// <param name="skipInputValidation">Whether to validate if the input is an RFC 8259-compliant JSON payload.</param>
84+
/// <exception cref="ArgumentException">Thrown if the length of the input is zero or equal to <see cref="int.MaxValue"/>.</exception>
85+
/// <exception cref="JsonException">
86+
/// Thrown if <paramref name="skipInputValidation"/> is <see langword="false"/>, and the input
87+
/// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259)
88+
/// or the input JSON exceeds a recursive depth of 64.
89+
/// </exception>
90+
/// <remarks>
91+
/// When writing untrused JSON values, do not set <paramref name="skipInputValidation"/> to <see langword="true"/> as this can result in invalid JSON
92+
/// being written, and/or the overall payload being written to the writer instance being invalid.
93+
///
94+
/// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled).
95+
///
96+
/// The <see cref="JsonWriterOptions.SkipValidation"/> value for the writer instance is honored when using this method.
97+
///
98+
/// The <see cref="JsonWriterOptions.Indented"/> and <see cref="JsonWriterOptions.Encoder"/> values for the writer instance are not applied when using this method.
99+
/// </remarks>
100+
public void WriteRawValue(ReadOnlySpan<byte> utf8Json, bool skipInputValidation = false)
101+
{
102+
if (!_options.SkipValidation)
103+
{
104+
ValidateWritingValue();
105+
}
106+
107+
if (utf8Json.Length == int.MaxValue)
108+
{
109+
ThrowHelper.ThrowArgumentException_ValueTooLarge(int.MaxValue);
110+
}
111+
112+
WriteRawValueCore(utf8Json, skipInputValidation);
113+
}
114+
115+
private void TranscodeAndWriteRawValue(ReadOnlySpan<char> json, bool skipInputValidation)
116+
{
117+
if (json.Length > JsonConstants.MaxUtf16RawValueLength)
118+
{
119+
ThrowHelper.ThrowArgumentException_ValueTooLarge(json.Length);
120+
}
121+
122+
byte[]? tempArray = null;
123+
124+
// For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold.
125+
Span<byte> utf8Json = json.Length <= (JsonConstants.ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ?
126+
// Use a pooled alloc.
127+
tempArray = ArrayPool<byte>.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) :
128+
// Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation)
129+
// and by using a normal alloc we can avoid the Clear().
130+
new byte[JsonReaderHelper.GetUtf8ByteCount(json)];
131+
132+
try
133+
{
134+
int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8Json);
135+
utf8Json = utf8Json.Slice(0, actualByteCount);
136+
WriteRawValueCore(utf8Json, skipInputValidation);
137+
}
138+
finally
139+
{
140+
if (tempArray != null)
141+
{
142+
utf8Json.Clear();
143+
ArrayPool<byte>.Shared.Return(tempArray);
144+
}
145+
}
146+
}
147+
148+
private void WriteRawValueCore(ReadOnlySpan<byte> utf8Json, bool skipInputValidation)
149+
{
150+
int len = utf8Json.Length;
151+
152+
if (len == 0)
153+
{
154+
ThrowHelper.ThrowArgumentException(SR.ExpectedJsonTokens);
155+
}
156+
157+
// In the UTF-16-based entry point methods above, we validate that the payload length <= int.MaxValue /3.
158+
// The result of this division will be rounded down, so even if every input character needs to be transcoded
159+
// (with expansion factor of 3), the resulting payload would be less than int.MaxValue,
160+
// as (int.MaxValue/3) * 3 is less than int.MaxValue.
161+
Debug.Assert(len < int.MaxValue);
162+
163+
if (skipInputValidation)
164+
{
165+
// Treat all unvalidated raw JSON value writes as string. If the payload is valid, this approach does
166+
// not affect structural validation since a string token is equivalent to a complete object, array,
167+
// or other complete JSON tokens when considering structural validation on subsequent writer calls.
168+
// If the payload is not valid, then we make no guarantees about the structural validation of the final payload.
169+
_tokenType = JsonTokenType.String;
170+
}
171+
else
172+
{
173+
// Utilize reader validation.
174+
Utf8JsonReader reader = new(utf8Json);
175+
while (reader.Read());
176+
_tokenType = reader.TokenType;
177+
}
178+
179+
// TODO (https://github.com/dotnet/runtime/issues/29293):
180+
// investigate writing this in chunks, rather than requesting one potentially long, contiguous buffer.
181+
int maxRequired = len + 1; // Optionally, 1 list separator. We've guarded against integer overflow earlier in the call stack.
182+
183+
if (_memory.Length - BytesPending < maxRequired)
184+
{
185+
Grow(maxRequired);
186+
}
187+
188+
Span<byte> output = _memory.Span;
189+
190+
if (_currentDepth < 0)
191+
{
192+
output[BytesPending++] = JsonConstants.ListSeparator;
193+
}
194+
195+
utf8Json.CopyTo(output.Slice(BytesPending));
196+
BytesPending += len;
197+
198+
SetFlagToAddListSeparatorBeforeNextItem();
199+
}
200+
}
201+
}

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@
187187
<Compile Include="Utf8JsonReaderTests.TryGet.Date.cs" />
188188
<Compile Include="Utf8JsonReaderTests.ValueTextEquals.cs" />
189189
<Compile Include="Utf8JsonWriterTests.cs" />
190+
<Compile Include="Utf8JsonWriterTests.WriteRaw.cs" />
190191
</ItemGroup>
191192
<ItemGroup>
192193
<Compile Include="..\..\src\System\Text\Json\BitStack.cs" Link="BitStack.cs" />

0 commit comments

Comments
 (0)