Interpreter block-count PGO for WebAssembly CoreCLR

src/coreclr/clrfeatures.cmake

@@ -13,6 +13,10 @@ if (FEATURE_DYNAMIC_CODE_COMPILED)
   set(FEATURE_PGO 1)
 endif()
 
+if (NOT DEFINED FEATURE_PGO AND CLR_CMAKE_TARGET_ARCH_WASM)
+  set(FEATURE_PGO 1)
+endif()
+
 # On desktop, if dynamic code compiled is false, we still enable static linking so we don't have to add platform manifest entries
 # for interpreter library, which is required for the packs build
 if (CLR_CMAKE_TARGET_ARCH_WASM OR CLR_CMAKE_TARGET_APPLE_MOBILE OR NOT FEATURE_DYNAMIC_CODE_COMPILED)

src/coreclr/inc/clrconfigvalues.h

@@ -514,6 +514,7 @@ CONFIG_DWORD_INFO(INTERNAL_OSR_HighId, W("OSR_HighId"), 10000000, "High end of e
 RETAIL_CONFIG_STRING_INFO(INTERNAL_PGODataPath, W("PGODataPath"), "Read/Write PGO data from/to the indicated file.")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ReadPGOData, W("ReadPGOData"), 0, "Read PGO data")
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_WritePGOData, W("WritePGOData"), 0, "Write PGO data")
+RETAIL_CONFIG_DWORD_INFO(INTERNAL_InterpPGO, W("InterpPGO"), 0, "Instrument interpreted methods with block counters and make the profile available for offline PGO (e.g. dotnet-pgo).")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_TieredPGO, W("TieredPGO"), 1, "Instrument Tier0 code and make counts available to Tier1")
 
 // TieredPGO_InstrumentOnlyHotCode values:

src/coreclr/interpreter/compiler.cpp

@@ -51,6 +51,9 @@ bool InterpCompiler::s_samplingProfilerEnabled = false;
 bool InterpCompiler::s_browserProfilerEnabled = false;
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+bool InterpCompiler::s_interpPgoEnabled = false;
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
 
 #if MEASURE_MEM_ALLOC
 #include <minipal/mutex.h>
@@ -2246,6 +2249,12 @@ InterpCompiler::InterpCompiler(COMP_HANDLE compHnd,
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
 
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+    m_emitInterpPGO = s_interpPgoEnabled
+        && (InterpConfig.InterpPGOMethods().isEmpty()
+            || InterpConfig.InterpPGOMethods().contains(compHnd, m_methodHnd, m_classHnd, &m_methodInfo->args));
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
+
 #ifdef DEBUG
     m_methodName = ::PrintMethodName(compHnd, m_classHnd, m_methodHnd, &m_methodInfo->args,
                             /* includeAssembly */ false,
@@ -2348,6 +2357,11 @@ bool InterpCompiler::CompileMethod()
     }
 #endif
 
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+    if (m_emitInterpPGO)
+        InstrumentBlockCounts();
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
+
     AllocOffsets();
     PatchInitLocals(m_methodInfo);
 
@@ -8668,6 +8682,87 @@ void InterpCompiler::CreateSynchronizedRetValVar()
     INTERP_DUMP("Created ret val var V%d\n", m_synchronizedOrAsyncRetValVarIndex);
 }
 
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+// Instrument each basic block with a block-count PGO probe. The counters are allocated by
+// allocPgoInstrumentationBySchema (native PgoManager memory), so they persist independently of
+// EventPipe session lifetime; the accumulated profile is flushed to the trace as
+// JitInstrumentationDataVerbose events, which dotnet-pgo consumes to build an .mibc.
+void InterpCompiler::InstrumentBlockCounts()
+{
+    // Mark blocks that are the target of a branch or switch (loop and branch heads). Together with
+    // the method entry, these are the block heads whose execution count can't be inferred from a
+    // single predecessor, so only they are worth a counter; the precompiler reconstructs the rest of
+    // the flow graph from them. This deliberately avoids the JIT's edge/spanning-tree scheme (#130517).
+    bool *isBranchTarget = getAllocator(IMK_BasicBlock).allocateZeroed<bool>(m_BBCount);
+    for (InterpBasicBlock *bb = m_pEntryBB; bb != NULL; bb = bb->pNextBB)
+    {
+        for (InterpInst *ins = bb->pFirstIns; ins != NULL; ins = ins->pNext)
+        {
+            if (ins->opcode == INTOP_SWITCH)
+            {
+                int32_t n = ins->data[0];
+                for (int32_t i = 0; i < n; i++)
+                    isBranchTarget[ins->info.ppTargetBBTable[i]->index] = true;
+            }
+            else if (InterpOpIsUncondBranch(ins->opcode) || InterpOpIsCondBranch(ins->opcode) ||
+                     ins->opcode == INTOP_LEAVE_CATCH || ins->opcode == INTOP_CALL_FINALLY)
+            {
+                isBranchTarget[ins->info.pTargetBB->index] = true;
+            }
+        }
+    }
+
+    // Collect the canonical block for each real IL offset that is the method entry (IL offset 0) or
+    // a branch/loop target. Clones (funclet / leave-chain islands) and blocks removed by optimization
+    // are skipped, so every schema entry carries a unique IL offset, matching what
+    // getPgoInstrumentationResults and dotnet-pgo expect.
+    TArray<InterpBasicBlock*, MemPoolAllocator> blocks(GetMemPoolAllocator(IMK_DataItem));
+    for (InterpBasicBlock *bb = m_pEntryBB; bb != NULL; bb = bb->pNextBB)
+    {
+        if (bb->ilOffset < 0 || bb->ilOffset >= m_ILCodeSizeFromILHeader || m_ppOffsetToBB[bb->ilOffset] != bb)
+            continue;
+        if (bb->ilOffset == 0 || isBranchTarget[bb->index])
+            blocks.Add(bb);
+    }
+
+    int32_t numBlocks = blocks.GetSize();
+    if (numBlocks == 0)
+        return;
+
+    // One 4-byte block counter per block.
+    TArray<ICorJitInfo::PgoInstrumentationSchema, MemPoolAllocator> schema(GetMemPoolAllocator(IMK_DataItem));
+    for (int32_t i = 0; i < numBlocks; i++)
+    {
+        ICorJitInfo::PgoInstrumentationSchema schemaElem;
+        schemaElem.Offset = 0;
+        schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::BasicBlockIntCount;
+        schemaElem.ILOffset = blocks.Get(i)->ilOffset;
+        schemaElem.Count = 1;
+        schemaElem.Other = 0;
+        schema.Add(schemaElem);
+    }
+
+    ICorJitInfo::PgoInstrumentationSchema *pSchema = schema.GetUnderlyingArray();
+    uint8_t *pInstrumentationData = NULL;
+    HRESULT hr = m_compHnd->allocPgoInstrumentationBySchema(m_methodHnd, pSchema, (uint32_t)numBlocks, &pInstrumentationData);
+    if (FAILED(hr) || pInstrumentationData == NULL)
+    {
+        INTERP_DUMP("InstrumentBlockCounts: allocPgoInstrumentationBySchema failed (hr=0x%08x)\n", hr);
+        return;
+    }
+
+    // Insert an INTOP_PGO_COUNT probe at the start of each block, pointing at its counter.
+    for (int32_t i = 0; i < numBlocks; i++)
+    {
+        uint32_t *pCounter = (uint32_t*)(pInstrumentationData + pSchema[i].Offset);
+        InterpInst *ins = InsertInsBB(blocks.Get(i), NULL, INTOP_PGO_COUNT);
+        // Probe is a pure counter increment with no IL mapping; keep it out of the debug maps.
+        ins->ilOffset = -1;
+        ins->data[0] = GetDataItemIndex((void*)pCounter);
+    }
+}
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
+
 void InterpCompiler::GenerateCode(CORINFO_METHOD_INFO* methodInfo)
 {
     bool readonly = false;

src/coreclr/interpreter/compiler.h

@@ -641,6 +641,10 @@ class InterpCompiler
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
 
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+    bool m_emitInterpPGO;
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
+
     void DeclarePointerIsClass(CORINFO_CLASS_HANDLE clsHnd)
     {
 #ifdef DEBUG
@@ -767,6 +771,10 @@ class InterpCompiler
     void CreateSynchronizedRetValVar();
 
     void GenerateCode(CORINFO_METHOD_INFO* methodInfo);
+
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+    void InstrumentBlockCounts();
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
     InterpBasicBlock* GenerateCodeForLeaveChainIslands(InterpBasicBlock *pNewBB, InterpBasicBlock *pPrevBB);
     void PatchInitLocals(CORINFO_METHOD_INFO* methodInfo);
 
@@ -1150,6 +1158,9 @@ class InterpCompiler
     static bool s_browserProfilerEnabled;
 #endif
 #endif // PERFTRACING_DISABLE_THREADS
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+    static bool s_interpPgoEnabled;
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
 
 #if MEASURE_MEM_ALLOC
     // Memory statistics for profiling.

src/coreclr/interpreter/eeinterp.cpp

@@ -41,6 +41,13 @@ extern "C" INTERP_API void jitStartup(ICorJitHost* jitHost)
 #endif // PERFTRACING_DISABLE_THREADS
     }
 
+    if (InterpConfig.InterpPGO() != 0)
+    {
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+        InterpCompiler::s_interpPgoEnabled = true;
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
+    }
+
     g_interpInitialized = true;
 }
 /*****************************************************************************/

src/coreclr/interpreter/inc/intops.def

@@ -90,6 +90,10 @@ OPDEF(INTOP_PROF_ENTER, "prof.enter", 2, 0, 0, InterpOpMethodHandle)
 OPDEF(INTOP_PROF_LEAVE, "prof.leave", 1, 0, 0, InterpOpNoArgs)
 #endif
 
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+OPDEF(INTOP_PGO_COUNT, "pgo.count", 2, 0, 0, InterpOpLdPtr)
+#endif
+
 OPDEF(INTOP_BR, "br", 2, 0, 0, InterpOpBranch)
 
 OPDEF(INTOP_BRFALSE_I4, "brfalse.i4", 3, 0, 1, InterpOpBranch)

src/coreclr/interpreter/interpconfigvalues.h

@@ -37,6 +37,8 @@ RELEASE_CONFIG_INTEGER(InterpMode, "InterpMode", 0); // Interpreter mode, one of
 
 RELEASE_CONFIG_INTEGER(DisplayMemStats, "JitMemStats", 0); // Display interpreter memory usage statistics (0=off, 1=summary, 2=detailed per-method)
 RELEASE_CONFIG_METHODSET(WasmPerformanceInstrumentation, "WasmPerformanceInstrumentation") // Method filter for WASM performance instrumentation profiler. Uses standard MethodSet pattern format.
+RELEASE_CONFIG_INTEGER(InterpPGO, "InterpPGO", 0); // Instrument interpreted methods with block counters and make the profile available for offline PGO (e.g. dotnet-pgo).
+RELEASE_CONFIG_METHODSET(InterpPGOMethods, "InterpPGOMethods") // Optional method filter scoping InterpPGO instrumentation (empty = all methods). Uses standard MethodSet pattern format.
 
 #undef CONFIG_STRING
 #undef RELEASE_CONFIG_STRING

src/coreclr/nativeaot/Runtime/eventpipe/ep-rt-aot.h

@@ -594,6 +594,16 @@ ep_rt_notify_profiler_provider_created (EventPipeProvider *provider)
     // Following mono's path of no-op
 }
 
+static
+inline
+void
+ep_rt_session_stopping (EventPipeSessionID session_id, uint64_t session_mask)
+{
+    // Following mono's path of no-op
+    (void)session_id;
+    (void)session_mask;
+}
+
 /*
  * Arrays.
  */

src/coreclr/vm/eventing/eventpipe/ep-rt-coreclr.cpp

@@ -6,9 +6,14 @@
 #ifdef ENABLE_PERFTRACING
 #include <eventpipe/ep-types.h>
 #include <eventpipe/ep.h>
+#include <eventpipe/ep-event.h>
+#include <eventpipe/ep-session.h>
 #include <eventpipe/ep-stack-contents.h>
 #include <eventpipe/ep-rt.h>
 #include "threadsuspend.h"
+#ifdef FEATURE_PGO
+#include "pgo.h"
+#endif
 
 ep_rt_lock_handle_t _ep_rt_coreclr_config_lock_handle;
 CrstStatic _ep_rt_coreclr_config_lock;
@@ -169,4 +174,47 @@ ep_rt_coreclr_sample_profiler_write_sampling_event_for_threads (
 	return;
 }
 
+void
+ep_rt_coreclr_session_stopping (EventPipeSessionID session_id, uint64_t session_mask)
+{
+	STATIC_CONTRACT_NOTHROW;
+#if defined(FEATURE_PGO) && defined(PERFTRACING_DISABLE_THREADS)
+	// Flush block-count PGO only into the session that enabled the JitInstrumentationData events
+	extern EventPipeEvent *EventPipeEventJitInstrumentationDataVerbose;
+	if (EventPipeEventJitInstrumentationDataVerbose != NULL &&
+		ep_event_is_enabled_by_mask (EventPipeEventJitInstrumentationDataVerbose, session_mask))
+	{
+		// Mark this thread as a rundown thread bound to the stopping session so the events emitted by the
+		// flush are routed to that single session (ep_session_write_event) instead of broadcast to every
+		// enabled session; the marker is cleared after the flush, including on exception. Dereferencing the
+		// session is safe here: this path is single-threaded (PERFTRACING_DISABLE_THREADS), so nothing frees
+		// it before section2 disables it.
+		EventPipeSession *session = reinterpret_cast<EventPipeSession *>(static_cast<uintptr_t>(session_id));
+		EventPipeThread *thread = ep_thread_get_or_create ();
+		if (thread != NULL)
+		{
+			ep_thread_set_as_rundown_thread (thread, session);
+			EX_TRY
+			{
+				PgoManager::EmitInstrumentationDataToEventPipe ();
+			}
+			EX_CATCH { }
+			EX_END_CATCH
+			ep_thread_set_as_rundown_thread (thread, NULL);
+		}
+	}
+#elif defined(FEATURE_PGO) && (defined(TARGET_BROWSER) || defined(TARGET_WASI))
+	extern EventPipeEvent *EventPipeEventJitInstrumentationDataVerbose;
+	if (EventPipeEventJitInstrumentationDataVerbose != NULL &&
+		ep_event_is_enabled_by_mask (EventPipeEventJitInstrumentationDataVerbose, session_mask))
+	{
+		PORTABILITY_ASSERT ("Interpreter block-count PGO flush is not implemented for multithreaded WASM (requires PERFTRACING_DISABLE_THREADS).");
+	}
+	(void)session_id;
+#else
+	(void)session_id;
+	(void)session_mask;
+#endif // FEATURE_PGO && PERFTRACING_DISABLE_THREADS
+}
+
 #endif /* ENABLE_PERFTRACING */

src/coreclr/vm/eventing/eventpipe/ep-rt-coreclr.h

@@ -659,6 +659,16 @@ ep_rt_notify_profiler_provider_created (EventPipeProvider *provider)
 #endif // !DACCESS_COMPILE && PROFILING_SUPPORTED
 }
 
+static
+inline
+void
+ep_rt_session_stopping (EventPipeSessionID session_id, uint64_t session_mask)
+{
+    STATIC_CONTRACT_NOTHROW;
+    extern void ep_rt_coreclr_session_stopping (EventPipeSessionID session_id, uint64_t session_mask);
+    ep_rt_coreclr_session_stopping (session_id, session_mask);
+}
+
 /*
  * Arrays.
  */
@@ -1093,8 +1103,13 @@ ep_rt_queue_job (
 	void *params)
 {
 #ifdef HOST_BROWSER
-	// In single-threaded mode the job runs on the browser event loop
-	SystemJS_DiagnosticServerQueueJob ((ep_rt_job_cb_t)job_func, params);
+	// In single-threaded mode the job runs on the browser event loop. Run the callback inline the
+	// first time so the diagnostic server makes progress synchronously (e.g. it can connect and
+	// resume during startup suspension) and only defer a re-schedule if it isn't done yet. Mirrors
+	// the Mono ep_rt_queue_job in ep-rt-mono.h.
+	ep_rt_job_cb_t cb = (ep_rt_job_cb_t)job_func;
+	if (!cb (params))
+		SystemJS_DiagnosticServerQueueJob (cb, params);
 	return true;
 #else
 	EP_UNREACHABLE ("Not implemented on this platform");

src/coreclr/vm/interpexec.cpp

@@ -2088,6 +2088,14 @@ void InterpExecMethod(InterpreterFrame *pInterpreterFrame, InterpMethodContextFr
                     INTOP_NEXT;
 #endif // TARGET_BROWSER && PERFTRACING_DISABLE_THREADS
 
+#if (defined(TARGET_BROWSER) || defined(TARGET_WASI)) && defined(PERFTRACING_DISABLE_THREADS)
+                INTOP_CASE(INTOP_PGO_COUNT)
+                    // Increment the block-count PGO counter whose address is stored as a data item.
+                    (*(uint32_t*)pMethod->pDataItems[ip[1]])++;
+                    ip += 2;
+                    INTOP_NEXT;
+#endif // (TARGET_BROWSER || TARGET_WASI) && PERFTRACING_DISABLE_THREADS
+
                 INTOP_CASE(INTOP_BR)
                     ip += ip[1];
                     INTOP_NEXT;

src/coreclr/vm/jitinterface.cpp

@@ -13061,7 +13061,19 @@ CORJIT_FLAGS* CEECodeGenInfo::getJitFlagsInternal()
 }
 
 /*********************************************************************/
-HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
+#ifdef FEATURE_PGO
+static bool InterpreterPgoInstrumentationEnabled()
+{
+#if defined(TARGET_BROWSER) || defined(TARGET_WASI)
+    static ConfigDWORD s_interpPgo;
+    return s_interpPgo.val(CLRConfig::INTERNAL_InterpPGO) != 0;
+#else
+    return false;
+#endif
+}
+#endif // FEATURE_PGO
+
+HRESULT CEECodeGenInfo::allocPgoInstrumentationBySchema(
             CORINFO_METHOD_HANDLE ftnHnd, /* IN */
             PgoInstrumentationSchema* pSchema, /* IN/OUT */
             uint32_t countSchemaItems, /* IN */
@@ -13080,9 +13092,10 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
 
 #ifdef FEATURE_PGO
 
-    // Only try instrumenting tiering-eligible methods
+    // Only try instrumenting tiering-eligible methods, unless interpreter PGO is enabled, in
+    // which case we instrument every method for offline profile collection.
     MethodDesc* pMD = (MethodDesc*)ftnHnd;
-    if (pMD->IsEligibleForTieredCompilation())
+    if (pMD->IsEligibleForTieredCompilation() || InterpreterPgoInstrumentationEnabled())
     {
         hr = PgoManager::allocPgoInstrumentationBySchema(pMD, m_ILHeader, pSchema, countSchemaItems, pInstrumentationData);
     }
@@ -13091,7 +13104,7 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
         hr = E_NOTIMPL;
     }
 #else
-    _ASSERTE(!"allocMethodBlockCounts not implemented on CEEJitInfo!");
+    _ASSERTE(!"allocMethodBlockCounts not implemented on CEECodeGenInfo!");
     hr = E_NOTIMPL;
 #endif // !FEATURE_PGO
 
@@ -13100,9 +13113,9 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema(
     return hr;
 }
 
-// Consider implementing getBBProfileData on CEEJitInfo.  This will allow us
+// Consider implementing getBBProfileData on CEECodeGenInfo.  This will allow us
 // to use profile info in codegen for non zapped images.
-HRESULT CEEJitInfo::getPgoInstrumentationResults(
+HRESULT CEECodeGenInfo::getPgoInstrumentationResults(
             CORINFO_METHOD_HANDLE      ftnHnd,
             PgoInstrumentationSchema **pSchema,                    // pointer to the schema table which describes the instrumentation results (pointer will not remain valid after jit completes)
             uint32_t *                 pCountSchemaItems,          // pointer to the count schema items
@@ -13163,7 +13176,7 @@ HRESULT CEEJitInfo::getPgoInstrumentationResults(
     *pPgoSource = pDataCur->m_pgoSource;
     hr = pDataCur->m_hr;
 #else
-    _ASSERTE(!"getPgoInstrumentationResults not implemented on CEEJitInfo!");
+    _ASSERTE(!"getPgoInstrumentationResults not implemented on CEECodeGenInfo!");
     hr = E_NOTIMPL;
 #endif