[JSC] The microtask queue aborts the process at 2^25 pending tasks

JSTests/stress/microtask-queue-more-than-2-25-tasks.js

@@ -0,0 +1,28 @@
+//@ skip if $memoryLimited
+//@ slow!
+//@ runDefault
+
+// The microtask queue was a WTF::Deque, and the 2^25th pending task made it grow to a capacity that is
+// not valid for a Vector, which aborts the process. This needs about 3 GB.
+
+function shouldBe(actual, expected)
+{
+    if (actual !== expected)
+        throw new Error(`bad value: expected ${expected} but got ${actual}`);
+}
+
+const count = 2 ** 25 + 1;
+const settled = Promise.resolve();
+let ran = 0;
+const job = () => { ++ran; };
+for (let i = 0; i < count; ++i)
+    settled.then(job);
+drainMicrotasks();
+shouldBe(ran, count);
+
+// The queue works again after it gave all of that back.
+let order = "";
+for (let i = 0; i < 5; ++i)
+    settled.then(() => { order += i; });
+drainMicrotasks();
+shouldBe(order, "01234");

JSTests/stress/microtask-queue-segments.js

@@ -0,0 +1,88 @@
+function shouldBe(actual, expected)
+{
+    if (actual !== expected)
+        throw new Error(`bad value: expected ${expected} but got ${actual}`);
+}
+
+// The microtask queue keeps its tasks in fixed-size segments: 409 tasks each with Bun's QueuedTask, 511
+// without. These depths stay inside one segment, end exactly at a segment boundary, or cross some.
+const depths = [0, 1, 2, 3, 100, 408, 409, 410, 510, 511, 512, 817, 818, 819, 1021, 1022, 1023, 1227, 1228, 5000];
+
+const settled = Promise.resolve();
+
+// Tasks run in the order in which they were queued, and the queue is usable again after it drains.
+for (const depth of depths) {
+    const order = [];
+    for (let i = 0; i < depth; ++i)
+        settled.then(() => { order.push(i); });
+    drainMicrotasks();
+    shouldBe(order.length, depth);
+    for (let i = 0; i < depth; ++i)
+        shouldBe(order[i], i);
+}
+
+// Every job queues its successor, so the queue keeps `depth` tasks while it moves forward through the
+// segments and frees them behind it. Only the queued task refers to each payload: a task that the
+// collector does not visit comes back with a dead payload.
+function walk(depth, jobsPerChain, collect)
+{
+    let ran = 0;
+    function makePayload(index, remaining)
+    {
+        return { index, remaining, text: `job ${index}`, filler: new Array(8).fill(index) };
+    }
+    function step(payload)
+    {
+        shouldBe(payload.index, ran);
+        shouldBe(payload.text, `job ${ran}`);
+        shouldBe(payload.filler.length, 8);
+        shouldBe(payload.filler[7], ran);
+        ++ran;
+        if (collect)
+            collect(ran);
+        if (payload.remaining)
+            Promise.resolve(makePayload(payload.index + depth, payload.remaining - 1)).then(step);
+    }
+    for (let chain = 0; chain < depth; ++chain)
+        Promise.resolve(makePayload(chain, jobsPerChain - 1)).then(step);
+    drainMicrotasks();
+    shouldBe(ran, depth * jobsPerChain);
+}
+
+for (const depth of [1, 2, 7, 408, 409, 410, 511, 512, 1300])
+    walk(depth, Math.ceil(4000 / depth), null);
+
+// Collections in the middle of a drain: full and eden, at segment boundaries and between them.
+walk(1, 400, (ran) => { if (!(ran % 50)) fullGC(); });
+walk(5, 300, (ran) => { if (!(ran % 101)) edenGC(); });
+walk(409, 8, (ran) => { if (!(ran % 409)) fullGC(); else if (!(ran % 200)) edenGC(); });
+walk(511, 8, (ran) => { if (!(ran % 511)) fullGC(); else if (!(ran % 250)) edenGC(); });
+walk(1000, 5, (ran) => { if (!(ran % 777)) gc(); });
+
+// A deep queue that is collected while it is full, while it drains, and while jobs add tasks to it.
+{
+    const depth = 20000;
+    let ran = 0;
+    let late = 0;
+    for (let i = 0; i < depth; ++i) {
+        Promise.resolve({ index: i, filler: new Array(4).fill(i) }).then((payload) => {
+            shouldBe(payload.index, ran);
+            shouldBe(payload.filler[3], ran);
+            ++ran;
+            if (!(ran % 4999))
+                fullGC();
+            if (!(ran % 1000)) {
+                Promise.resolve({ queuedAt: ran, filler: new Array(4).fill(ran) }).then((payload) => {
+                    shouldBe(ran, depth);
+                    shouldBe(payload.filler[0], payload.queuedAt);
+                    ++late;
+                });
+            }
+        });
+    }
+    fullGC();
+    edenGC();
+    drainMicrotasks();
+    shouldBe(ran, depth);
+    shouldBe(late, depth / 1000);
+}

Source/JavaScriptCore/runtime/MicrotaskQueue.cpp

@@ -138,13 +138,88 @@ void MicrotaskQueue::enqueueSlow(QueuedTask&& task)
         scheduleToRunIfNeeded();
 }
 
-bool MarkedMicrotaskDeque::hasMicrotasksForFullyActiveDocument() const
+void MarkedMicrotaskDeque::appendSegment()
 {
-    for (auto& task : m_queue) {
-        if (task.isRunnable())
-            return true;
+    auto* segment = new Segment;
+    if (m_tail)
+        m_tail->next = segment;
+    else {
+        m_head = segment;
+        m_front = segment->begin();
     }
-    return false;
+    m_tail = segment;
+    m_back = segment->begin();
+    m_backLimit = segment->end();
+    ++m_segmentCount;
+}
+
+void MarkedMicrotaskDeque::removeHeadSegment()
+{
+    // The queue is not empty, so the tail is a later segment.
+    auto* segment = m_head;
+    m_head = segment->next;
+    m_front = m_head->begin();
+    --m_segmentCount;
+    delete segment;
+}
+
+void MarkedMicrotaskDeque::clear()
+{
+    while (auto* segment = m_head) {
+        m_head = segment->next;
+        delete segment;
+    }
+    m_tail = nullptr;
+    m_front = nullptr;
+    m_back = nullptr;
+    m_backLimit = nullptr;
+    m_segmentCount = 0;
+    m_markedBefore = 0;
+}
+
+template<typename Functor>
+ALWAYS_INLINE void MarkedMicrotaskDeque::forEachTaskAfter(size_t toSkip, const Functor& functor) const
+{
+    ASSERT(toSkip <= size());
+#if ASSERT_ENABLED
+    size_t remaining = size() - toSkip;
+#endif
+    auto* segment = m_head;
+    auto* task = m_front;
+    while (toSkip) {
+        size_t count = std::min(toSkip, static_cast<size_t>(segment->end() - task));
+        task += count;
+        toSkip -= count;
+        if (toSkip) {
+            segment = segment->next;
+            task = segment->begin();
+        }
+    }
+    for (; task != m_back; ++task) {
+        if (task == segment->end()) {
+            segment = segment->next;
+            task = segment->begin();
+        }
+#if ASSERT_ENABLED
+        ASSERT(remaining);
+        --remaining;
+#endif
+        if (functor(*task) == IterationStatus::Done)
+            return;
+    }
+    ASSERT(!remaining);
+}
+
+bool MarkedMicrotaskDeque::hasMicrotasksForFullyActiveDocument() const
+{
+    bool result = false;
+    forEachTaskAfter(0, [&](QueuedTask& task) {
+        if (!task.isRunnable())
+            return IterationStatus::Continue;
+        result = true;
+        return IterationStatus::Done;
+    });
+    return result;
 }
 
 template<typename Visitor>
@@ -158,12 +233,12 @@ void MarkedMicrotaskDeque::visitAggregateImpl(Visitor& visitor)
     // This cursor is adjusted when an entry is dequeued. And we do not use any locking here, and that's fine: these
     // values are read by GC when CollectorPhase::FixPoint and CollectorPhase::Begin, and both suspend the mutator, thus,
     // there is no concurrency issue.
-    for (auto iterator = m_queue.begin() + m_markedBefore, end = m_queue.end(); iterator != end; ++iterator) {
-        auto& task = *iterator;
+    forEachTaskAfter(m_markedBefore, [&](QueuedTask& task) {
         visitor.appendUnbarriered(task.dispatcher());
         visitor.appendUnbarriered(task.m_arguments, QueuedTask::maxArguments);
-    }
-    m_markedBefore = m_queue.size();
+        return IterationStatus::Continue;
+    });
+    m_markedBefore = size();
 }
 DEFINE_VISIT_AGGREGATE(MarkedMicrotaskDeque);
 

Source/JavaScriptCore/runtime/MicrotaskQueue.h

@@ -30,10 +30,13 @@
 #include "SlotVisitorMacros.h"
 #include <wtf/CompactPointerTuple.h>
 #include <wtf/Compiler.h>
-#include <wtf/Deque.h>
+#include <wtf/FastMalloc.h>
+#include <wtf/IterationStatus.h>
+#include <wtf/Noncopyable.h>
 #include <wtf/Ref.h>
 #include <wtf/RefCounted.h>
 #include <wtf/SentinelLinkedList.h>
+#include <wtf/StdLibExtras.h>
 #include <wtf/TZoneMalloc.h>
 #include <wtf/VectorTraits.h>
 
@@ -154,39 +157,62 @@ static_assert(sizeof(QueuedTask) <= 32, "Size of QueuedTask is critical for perf
 #endif
 static_assert(std::is_trivially_destructible_v<QueuedTask>);
 
+// A FIFO of QueuedTasks, stored in a singly linked list of fixed-size segments. A WTF::Deque keeps its
+// tasks in one buffer that doubles, and a buffer of 2^26 tasks is not a valid Vector capacity, so the
+// 2^25th pending task aborted the process. Script reaches that count (one promise with that many
+// reactions is enough), and enqueue() has no way to report a failure. With segments no allocation
+// grows with the queue, a task never moves, and a segment is freed when its last task is dequeued.
 class MarkedMicrotaskDeque {
+    WTF_MAKE_NONCOPYABLE(MarkedMicrotaskDeque);
 public:
     friend class MicrotaskQueue;
 
     MarkedMicrotaskDeque() = default;
+    ~MarkedMicrotaskDeque() { clear(); }
 
-    const QueuedTask& front() const LIFETIME_BOUND { return m_queue.first(); }
+    const QueuedTask& front() const LIFETIME_BOUND
+    {
+        ASSERT(!isEmpty());
+        return *m_front;
+    }
 
+    // This can free the segment that front() points into.
     QueuedTask dequeue()
     {
+        ASSERT(!isEmpty());
         if (m_markedBefore)
             --m_markedBefore;
-        return m_queue.takeFirst();
+        QueuedTask task = WTF::move(*m_front);
+        ++m_front;
+        if (m_front == m_back) {
+            // Empty. Start over at the first slot, so that a shallow queue stays in the same cache lines.
+            m_front = m_back = m_tail->begin();
+        } else if (m_front == m_head->end()) [[unlikely]]
+            removeHeadSegment();
+        return task;
     }
 
     void enqueue(QueuedTask&& task)
     {
-        m_queue.append(WTF::move(task));
-    }
-
-    bool isEmpty() const
-    {
-        return m_queue.isEmpty();
+        if (m_back == m_backLimit) [[unlikely]]
+            appendSegment();
+        new (NotNull, m_back) QueuedTask(WTF::move(task));
+        ++m_back;
     }
 
-    size_t size() const { return m_queue.size(); }
+    // m_back can be one past the end of its segment. That address is never a slot of another segment,
+    // because a segment's slots start after its header.
+    bool isEmpty() const { return m_front == m_back; }
 
-    void clear()
+    size_t size() const
     {
-        m_queue.clear();
-        m_markedBefore = 0;
+        if (!m_segmentCount)
+            return 0;
+        return (m_segmentCount - 1) * Segment::capacity + (m_back - m_tail->begin()) - (m_front - m_head->begin());
     }
 
+    JS_EXPORT_PRIVATE void clear();
+
 #if USE(BUN_JSC_ADDITIONS)
     // Defined in MicrotaskQueueInlines.h (requires globalObject() from QueuedTask).
     inline void clearForGlobalObject(JSGlobalObject* targetGlobalObject);
@@ -199,7 +225,12 @@ class MarkedMicrotaskDeque {
 
     void swap(MarkedMicrotaskDeque& other)
     {
-        m_queue.swap(other.m_queue);
+        std::swap(m_front, other.m_front);
+        std::swap(m_back, other.m_back);
+        std::swap(m_backLimit, other.m_backLimit);
+        std::swap(m_head, other.m_head);
+        std::swap(m_tail, other.m_tail);
+        std::swap(m_segmentCount, other.m_segmentCount);
         std::swap(m_markedBefore, other.m_markedBefore);
     }
 
@@ -208,7 +239,30 @@ class MarkedMicrotaskDeque {
     DECLARE_VISIT_AGGREGATE;
 
 private:
-    Deque<QueuedTask> m_queue;
+    struct Segment {
+        WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(Segment);
+
+        static constexpr size_t capacity = (16 * KB - sizeof(Segment*)) / sizeof(QueuedTask);
+
+        QueuedTask* begin() { return reinterpret_cast<QueuedTask*>(storage); }
+        QueuedTask* end() { return begin() + capacity; }
+
+        Segment* next { nullptr };
+        alignas(QueuedTask) std::byte storage[capacity * sizeof(QueuedTask)];
+    };
+
+    JS_EXPORT_PRIVATE void appendSegment();
+    JS_EXPORT_PRIVATE void removeHeadSegment();
+
+    // Calls the functor with each task in queue order, except for the first toSkip tasks.
+    template<typename Functor> void forEachTaskAfter(size_t toSkip, const Functor&) const;
+
+    QueuedTask* m_front { nullptr }; // The next task to dequeue, in m_head.
+    QueuedTask* m_back { nullptr }; // The slot for the next task to enqueue, in m_tail.
+    QueuedTask* m_backLimit { nullptr }; // m_tail->end(), or null before the first segment exists.
+    Segment* m_head { nullptr };
+    Segment* m_tail { nullptr };
+    size_t m_segmentCount { 0 };
     size_t m_markedBefore { 0 };
 };
 

Source/JavaScriptCore/runtime/MicrotaskQueueInlines.h

@@ -79,14 +79,13 @@ inline void MarkedMicrotaskDeque::clearForGlobalObject(JSGlobalObject* targetGlo
 {
     if (!targetGlobalObject)
         return;
-    Deque<QueuedTask> remaining;
-    while (!m_queue.isEmpty()) {
-        QueuedTask task = m_queue.takeFirst();
+    MarkedMicrotaskDeque remaining;
+    while (!isEmpty()) {
+        QueuedTask task = dequeue();
         if (task.globalObject() != targetGlobalObject)
-            remaining.append(WTF::move(task));
+            remaining.enqueue(WTF::move(task));
     }
-    m_queue.swap(remaining);
-    m_markedBefore = 0;
+    swap(remaining);
 }
 
 inline void MicrotaskQueue::clearForGlobalObject(JSGlobalObject* targetGlobalObject)