Implement til::au16 and til::u16a conversion functions & make first use in WriteConsoleAImpl

.github/actions/spell-check/whitelist/whitelist.txt

@@ -114,6 +114,7 @@ ATest
 atg
 attr
 ATTRCOLOR
+au
 aumid
 Authenticode
 AUTOBUDDY
@@ -976,6 +977,7 @@ Greyscale
 gridline
 groupbox
 gsl
+GSM
 GTP
 guc
 gui

src/cascadia/TerminalConnection/ConptyConnection.h

@@ -53,7 +53,7 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
         wil::unique_static_pseudoconsole_handle _hPC;
         wil::unique_threadpool_wait _clientExitWait;
 
-        til::u8state _u8State;
+        til::astate _u8State;
         std::wstring _u16Str;
         std::array<char, 4096> _buffer;
 

src/host/VtInputThread.hpp

@@ -39,6 +39,6 @@ namespace Microsoft::Console
         HRESULT _exitResult;
 
         std::unique_ptr<Microsoft::Console::VirtualTerminal::StateMachine> _pInputStateMachine;
-        til::u8state _u8State;
+        til::astate _u8State;
     };
 }

src/host/_stream.cpp

@@ -1082,94 +1082,18 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
         auto unlock{ wil::scope_exit([&] { UnlockConsole(); }) };
 
         auto& screenInfo{ context.GetActiveBuffer() };
-        const auto& consoleInfo{ ServiceLocator::LocateGlobals().getConsoleInformation() };
-        const auto codepage{ consoleInfo.OutputCP };
-        auto leadByteCaptured{ false };
-        auto leadByteConsumed{ false };
+        const auto codepage{ ServiceLocator::LocateGlobals().getConsoleInformation().OutputCP };
         std::wstring wstr{};
-        static til::u8state u8State{};
 
         // Convert our input parameters to Unicode
+        const auto leadByteConsumed{ !screenInfo.aState.empty(codepage) }; // only used for DBCS
+        RETURN_IF_FAILED(til::au16(codepage, buffer, wstr, screenInfo.aState));
+        const auto leadByteCaptured{ !screenInfo.aState.empty(codepage) }; // only used for DBCS
+
         if (codepage == CP_UTF8)
         {
-            RETURN_IF_FAILED(til::u8u16(buffer, wstr, u8State));
             read = buffer.size();
         }
-        else
-        {
-            // In case the codepage changes from UTF-8 to another,
-            // we discard partials that might still be cached.
-            u8State.reset();
-
-            int mbPtrLength{};
-            RETURN_IF_FAILED(SizeTToInt(buffer.size(), &mbPtrLength));
-
-            // (buffer.size() + 2) I think because we might be shoving another unicode char
-            // from screenInfo->WriteConsoleDbcsLeadByte in front
-            // because we previously checked that buffer.size() fits into an int, +2 won't cause an overflow of size_t
-            wstr.resize(buffer.size() + 2);
-
-            wchar_t* wcPtr{ wstr.data() };
-            auto mbPtr{ buffer.data() };
-            size_t dbcsLength{};
-            if (screenInfo.WriteConsoleDbcsLeadByte[0] != 0 && gsl::narrow_cast<byte>(*mbPtr) >= byte{ ' ' })
-            {
-                // there was a portion of a dbcs character stored from a previous
-                // call so we take the 2nd half from mbPtr[0], put them together
-                // and write the wide char to wcPtr[0]
-                screenInfo.WriteConsoleDbcsLeadByte[1] = gsl::narrow_cast<byte>(*mbPtr);
-
-                try
-                {
-                    const auto wFromComplemented{
-                        ConvertToW(codepage, { reinterpret_cast<const char*>(screenInfo.WriteConsoleDbcsLeadByte), ARRAYSIZE(screenInfo.WriteConsoleDbcsLeadByte) })
-                    };
-
-                    FAIL_FAST_IF(wFromComplemented.size() != 1);
-                    dbcsLength = sizeof(wchar_t);
-                    wcPtr[0] = wFromComplemented.at(0);
-                    mbPtr++;
-                }
-                catch (...)
-                {
-                    dbcsLength = 0;
-                }
-
-                // this looks weird to be always incrementing even if the conversion failed, but this is the
-                // original behavior so it's left unchanged.
-                wcPtr++;
-                mbPtrLength--;
-
-                // Note that we used a stored lead byte from a previous call in order to complete this write
-                // Use this to offset the "number of bytes consumed" calculation at the end by -1 to account
-                // for using a byte we had internally, not off the stream.
-                leadByteConsumed = true;
-            }
-
-            screenInfo.WriteConsoleDbcsLeadByte[0] = 0;
-
-            // if the last byte in mbPtr is a lead byte for the current code page,
-            // save it for the next time this function is called and we can piece it
-            // back together then
-            if (mbPtrLength != 0 && CheckBisectStringA(const_cast<char*>(mbPtr), mbPtrLength, &consoleInfo.OutputCPInfo))
-            {
-                screenInfo.WriteConsoleDbcsLeadByte[0] = gsl::narrow_cast<byte>(mbPtr[mbPtrLength - 1]);
-                mbPtrLength--;
-
-                // Note that we captured a lead byte during this call, but won't actually draw it until later.
-                // Use this to offset the "number of bytes consumed" calculation at the end by +1 to account
-                // for taking a byte off the stream.
-                leadByteCaptured = true;
-            }
-
-            if (mbPtrLength != 0)
-            {
-                // convert the remaining bytes in mbPtr to wide chars
-                mbPtrLength = sizeof(wchar_t) * MultiByteToWideChar(codepage, 0, mbPtr, mbPtrLength, wcPtr, mbPtrLength);
-            }
-
-            wstr.resize((dbcsLength + mbPtrLength) / sizeof(wchar_t));
-        }
 
         // Hold the specific version of the waiter locally so we can tinker with it if we must to store additional context.
         std::unique_ptr<WriteData> writeDataWaiter{};

src/host/screenInfo.cpp

@@ -40,10 +40,9 @@ SCREEN_INFORMATION::SCREEN_INFORMATION(
     HWheelDelta{ 0 },
     _textBuffer{ nullptr },
     Next{ nullptr },
-    WriteConsoleDbcsLeadByte{ 0, 0 },
-    FillOutDbcsLeadChar{ 0 },
     ConvScreenInfo{ nullptr },
     ScrollScale{ 1ul },
+    aState{},
     _pConsoleWindowMetrics{ pMetrics },
     _pAccessibilityNotifier{ pNotifier },
     _stateMachine{ nullptr },

src/host/screenInfo.hpp

@@ -172,14 +172,14 @@ class SCREEN_INFORMATION : public ConsoleObjectHeader, public Microsoft::Console
 
 public:
     SCREEN_INFORMATION* Next;
-    BYTE WriteConsoleDbcsLeadByte[2];
-    BYTE FillOutDbcsLeadChar;
 
     // non ownership pointer
     ConversionAreaInfo* ConvScreenInfo;
 
     UINT ScrollScale;
 
+    til::astate aState{};
+
     bool IsActiveScreenBuffer() const;
 
     const Microsoft::Console::VirtualTerminal::StateMachine& GetStateMachine() const;

src/inc/til/at.h

@@ -3,19 +3,48 @@
 
 #pragma once
 
+// The at function declares that you've already sufficiently checked that your array access
+// is in range before retrieving an item inside it at an offset.
+// This is to save double/triple/quadruple testing in circumstances where you are already
+// pivoting on the length of a set and now want to pull elements out of it by offset
+// without checking again.
+// gsl::at will do the check again. As will .at(). And using [] will have a warning in audit.
+
 namespace til
 {
-    // The at function declares that you've already sufficiently checked that your array access
-    // is in range before retrieving an item inside it at an offset.
-    // This is to save double/triple/quadruple testing in circumstances where you are already
-    // pivoting on the length of a set and now want to pull elements out of it by offset
-    // without checking again.
-    // gsl::at will do the check again. As will .at(). And using [] will have a warning in audit.
-    template<class T>
-    constexpr auto at(T& cont, const size_t i) -> decltype(cont[cont.size()])
+    // Routine Description:
+    // - Takes a reference to a sequence of constant data and returns the item at the given offset.
+    //   NOTE: The function relies on a sufficient check that the access is in range.
+    // Arguments:
+    // - sequence - Reference to constant data in a range of data accessible using the subscript operator.
+    //              Such like constant STL strings, containers, bitsets, random-access iterators, or pointers.
+    // - index    - Offset of the element to be returned.
+    // Return Value:
+    // - Element in the sequence at the offset given by the index parameter.
+    template<class T, class U>
+    constexpr auto at(const T& sequence, const U index) -> typename std::enable_if<std::is_integral<U>::value, decltype(sequence[0])>::type
+    {
+#pragma warning(push)
+#pragma warning(suppress : 26481 26482 26446) // Suppress checks for pointer arithmetic, indexing with constant expressions, and subscript operator.
+        return sequence[index];
+#pragma warning(pop)
+    }
+
+    // Routine Description:
+    // - Takes a reference to a sequence of data and returns the item at the given offset.
+    //   NOTE: The function relies on a sufficient check that the access is in range.
+    // Arguments:
+    // - sequence - Reference to constant data in a range of data accessible using the subscript operator.
+    //              Such like STL strings, containers, bitsets, random-access iterators, or pointers.
+    // - index    - Offset of the element to be returned.
+    // Return Value:
+    // - Element in the sequence at the offset given by the index parameter.
+    template<class T, class U>
+    constexpr auto at(T& sequence, const U index) -> typename std::enable_if<std::is_integral<U>::value, decltype(sequence[0])>::type
     {
-#pragma warning(suppress : 26482) // Suppress bounds.2 check for indexing with constant expressions
-#pragma warning(suppress : 26446) // Suppress bounds.4 check for subscript operator.
-        return cont[i];
+#pragma warning(push)
+#pragma warning(suppress : 26481 26482 26446) // Suppress checks for pointer arithmetic, indexing with constant expressions, and subscript operator.
+        return sequence[index];
+#pragma warning(pop)
     }
 }