[RISC-V] "Unsupported CPU type" exception on riscv64 host

[dkurt]

• Tested on x86 Ubuntu 20.04

An exception https://github.com/llvm/llvm-project/blob/de9d80c1c579e39cc658a508f1d4ba1cd792e4d5/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp#L1076 appears when trying run cross-compiled Halide on RISC-V QEMU machine.

Unsupported CPU type!
UNREACHABLE executed at /home/dkurt/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp:1076!
Aborted

Application code

#include "Halide.h"

using namespace Halide;

int main(int argc, char** argv) {
    Func brighter;
    Var x, y;

    Param<uint8_t> myoffset("myoffset");
    ImageParam input(type_of<uint8_t>(), 2);
    std::vector<Argument> args(2);
    args[0] = input;
    args[1] = myoffset;

    brighter(x, y) = input(x, y) * myoffset;

    brighter.bound(x, 0, 64).bound(y, 0, 64);

    Target target = get_host_target();
    std::cout << target << std::endl;

    brighter.print_loop_nest();

    Buffer<uint8_t> output(64, 64);
    brighter.realize(output, target);
}

include_directories(
    /home/dkurt/halide-install/include/
)

link_directories(
    /home/dkurt/halide-install/lib
)

set(CMAKE_C_COMPILER riscv64-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER riscv64-linux-gnu-g++)

set(CMAKE_CXX_FLAGS "-std=c++17")
add_executable(app main.cpp)
target_link_libraries(app Halide)

export LD_LIBRARY_PATH=/usr/riscv64-linux-gnu/lib/:$LD_LIBRARY_PATH
qemu-riscv64-static ./app

Steps to reproduce:

1. Install toolchain

sudo apt-get install -y qemu-user-static crossbuild-essential-riscv64

3. Build LLVM

riscv64.toolchain.cmake

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR riscv64)

set(CMAKE_C_COMPILER riscv64-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER riscv64-linux-gnu-g++)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

find_program(QEMU_RISCV64 qemu-riscv64-static)
if (QEMU_RISCV64)
    set(CMAKE_CROSSCOMPILING_EMULATOR ${QEMU_RISCV64})
endif ()

git clone --depth 1 --branch llvmorg-15.0.2 https://github.com/llvm/llvm-project.git

cmake -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_TOOLCHAIN_FILE=$(realpath riscv64.toolchain.cmake) \
      -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra" \
      -DLLVM_TARGETS_TO_BUILD="RISCV" \
      -DLLVM_TARGET_ARCH="riscv64" \
      -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_ASSERTIONS=ON \
      -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_BUILD_32_BITS=OFF \
      -S llvm-project/llvm -B llvm-build

cmake --build llvm-build -j$(nproc --all)
cmake --install llvm-build --prefix llvm-install

4. Clone Halide (use custom branch as recommended in Ramp of lanes <= 1 exception when cross-compile Halide for RISC-V #7070)

git clone --depth 1 -b riscv_update https://github.com/halide/Halide

5. Apply Halide patch (see discussion at Fix Halide cross-compilation #7073)

patch

index cc9f6805b..9b3c25751 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -541,7 +541,7 @@ endif ()
 
 if (BUILD_SHARED_LIBS)
     message(STATUS "Building autoschedulers enabled")
-    add_subdirectory(autoschedulers)
+    # add_subdirectory(autoschedulers)
 else ()
     message(STATUS "Building autoschedulers disabled (static Halide)")
 endif ()
diff --git a/src/runtime/CMakeLists.txt b/src/runtime/CMakeLists.txt
index b4888c82e..a1ae1e205 100644
--- a/src/runtime/CMakeLists.txt
+++ b/src/runtime/CMakeLists.txt
@@ -243,14 +243,14 @@ foreach (i IN LISTS RUNTIME_CPP)
                 target_compile_definitions(${basename} PRIVATE ${RUNTIME_DEFINES})
             else()
                 add_custom_command(OUTPUT "${LL}"
-                                   COMMAND ${CMAKE_C_COMPILER_LAUNCHER} $<TARGET_FILE:clang> ${clang_flags} -o "${LL}" "$<SHELL_PATH:${SOURCE}>"
+                                   COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} ${CMAKE_C_COMPILER_LAUNCHER} $<TARGET_FILE:clang> ${clang_flags} -o "${LL}" "$<SHELL_PATH:${SOURCE}>"
                                    DEPENDS "${SOURCE}"
                                    DEPFILE "${basename}.d"
                                    VERBATIM)
             endif()
 
             add_custom_command(OUTPUT "${BC}"
-                               COMMAND llvm-as "${LL}" -o "${BC}"
+                               COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:llvm-as> "${LL}" -o "${BC}"
                                DEPENDS "${LL}"
                                VERBATIM)
 
@@ -294,7 +294,7 @@ foreach (i IN LISTS RUNTIME_LL)
     endif ()
 
     add_custom_command(OUTPUT "${BC}"
-                       COMMAND llvm-as "${LL_TRANSFORMED}" -o "${BC}"
+                       COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:llvm-as> "${LL_TRANSFORMED}" -o "${BC}"
                        DEPENDS "${LL_TRANSFORMED}"
                        VERBATIM)
     add_custom_command(OUTPUT "${INITMOD}"

5. Build Halide

sudo ln -s /usr/riscv64-linux-gnu/lib/ld-linux-riscv64-lp64d.so.1 /lib/ld-linux-riscv64-lp64d.so.1

cmake -DLLVM_DIR=$(realpath llvm-install/lib/cmake/llvm) \
      -DClang_DIR=$(realpath llvm-install/lib/cmake/clang) \
      -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_TOOLCHAIN_FILE=$(realpath riscv64.toolchain.cmake) \
      -DWITH_TESTS=OFF \
      -DWITH_TUTORIALS=OFF \
      -DWITH_PYTHON_BINDINGS=OFF \
      -S Halide -B halide-build

cmake --build halide-build -j$(nproc --all)
cmake --install halide-build --prefix halide-install

[dkurt]

Same problem on real hardware MangoPi. So this it not QEMU issue.

[zvookin]

Can you please provide your target flags for the Halide compile? Are you using JIT or AOT compilation here?

[zvookin]

Also, if possible can you cat /proc/cpuinfo on the device and put the output on this issue or somewhere? This is assuming running Linux etc. I can't find easily parseable detail on which extensions the C906 supports and which might be configurable for the implementation of the core on a specific SoC. I'm hoping CPU info will say exactly what it supports and we can human diff that against what is being asked for in CodeGen_RISCV::mattrs() . I assume it has a vector unit and you set vector_bits_128 in the target.

[dkurt]

@zvookin, target flags from get_host_target():

target(riscv-64-linux)

Output from cat /proc/cpuinfo:

processor       : 0
hart            : 0
isa             : rv64imafdc
mmu             : sv39
uarch           : thead,c906

Please note that problem appears even with not vectorized scheduler.

[zvookin]

EDIT: just saw the target here. This is probably not the issue....

This is happening before even running code. Are you specifying "rvv" in the target? If so, we are requesting the "v" extension and it looks like that is not available. Is it still crashing if you remove "rvv" and "vector_bits_128" from the target? (And of course not using any vectorization in the schedule, though that might just scalarize if no vector option is available.)

[zvookin]

Is this trying to use JIT? It looks like maybe LLVM doesn't support that. Might be some flags that can be added to control the type of relocations.

[dkurt]

@zvookin, oh, yes, this is about JIT. AOT compilation works fine. I will check on LLVM side, thanks!

[dkurt]

Might the problem be in using old RuntimeDyld rather than new JITLink (see llvm/llvm-project#58652 (comment)) ?

Halide/src/JITModule.cpp

Line 270 in fac313e

engine_builder.setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager>(memory_manager));