[RISCV] Unsupported CPU type at RuntimeDyldELF::resolveRelocation

[dkurt]

Hi there!

I'm trying compile and run Halide project based on LLVM on riscv64. Specifically for Halide, there are two options: Ahead-Of-Time (AOT) and JIT. AOT works fine when I compile on x86 and use riscv64 binaries in QEMU or T-Head C906 CPU. Unfortunately, for JIT pipeline, there is an error "Unsupported CPU type!" at RuntimeDyldELF::resolveRelocation:

llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp

Line 1076 in de9d80c

llvm_unreachable("Unsupported CPU type!");

Not sure that it might be helpful, but these are steps to reproduce:

Steps to reproduce

Application for test:

/opt/riscv/bin/riscv64-unknown-linux-gnu-g++ -I$HOME/halide_install_riscv64/include/ -L$HOME/halide_install_riscv64/lib/ -lHalide ./main.cpp -o app
export LD_LIBRARY_PATH=$HOME/halide_install_riscv64/lib/:$LD_LIBRARY_PATH
/opt/riscv/bin/qemu-riscv64 ./app

Application code

#include "Halide.h"

using namespace Halide;

int main(int argc, char** argv) {
    Func brighter;
    Var x, y;

    Param<uint8_t> myoffset("myoffset");
    ImageParam input(type_of<uint8_t>(), 2);
    std::vector<Argument> args(2);
    args[0] = input;
    args[1] = myoffset;

    brighter(x, y) = input(x, y) * myoffset;

    brighter.bound(x, 0, 64).bound(y, 0, 64);

    Target target = get_host_target();
    std::cout << target << std::endl;

    std::vector<Target::Feature> features;
    target.set_features(features);

    brighter.print_loop_nest();

    Buffer<uint8_t> output(64, 64);
    brighter.realize(output);
}

1. Prepare toolchain

git clone --depth 1 https://github.com/riscv-collab/riscv-gnu-toolchain
./configure --prefix=/opt/riscv
sudo make linux -j$(nproc --all)
sudo make build-qemu -j$(nproc --all)

2. Build LLVM

riscv64.toolchain.cmake

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR riscv64)

set(CMAKE_C_COMPILER riscv64-unknown-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER riscv64-unknown-linux-gnu-g++)

set(CMAKE_SYSROOT /opt/riscv/sysroot CACHE PATH "RISC-V sysroot")

set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

find_program(QEMU_RISCV64 qemu-riscv64)
if (QEMU_RISCV64)
    set(CMAKE_CROSSCOMPILING_EMULATOR ${QEMU_RISCV64})
endif ()

git clone --depth 1 --branch llvmorg-15.0.2 https://github.com/llvm/llvm-project.git

export PATH=/opt/riscv/bin:$PATH

cmake -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_TOOLCHAIN_FILE=$(realpath riscv64.toolchain.cmake) \
      -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra" \
      -DLLVM_TARGETS_TO_BUILD="RISCV" \
      -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_ASSERTIONS=ON \
      -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_BUILD_32_BITS=OFF \
      -S llvm-project/llvm -B llvm-build

cmake --build llvm-build -j$(nproc --all)
cmake --install llvm-build --prefix llvm-install

3. Clone Halide (use custom branch as recommended in Ramp of lanes <= 1 exception when cross-compile Halide for RISC-V halide/Halide#7070)

git clone --depth 1 -b riscv_update https://github.com/halide/Halide

5. Apply Halide patch (see discussion at Fix Halide cross-compilation halide/Halide#7073)

patch

index cc9f6805b..9b3c25751 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -541,7 +541,7 @@ endif ()
 
 if (BUILD_SHARED_LIBS)
     message(STATUS "Building autoschedulers enabled")
-    add_subdirectory(autoschedulers)
+    # add_subdirectory(autoschedulers)
 else ()
     message(STATUS "Building autoschedulers disabled (static Halide)")
 endif ()
diff --git a/src/runtime/CMakeLists.txt b/src/runtime/CMakeLists.txt
index b4888c82e..a1ae1e205 100644
--- a/src/runtime/CMakeLists.txt
+++ b/src/runtime/CMakeLists.txt
@@ -243,14 +243,14 @@ foreach (i IN LISTS RUNTIME_CPP)
                 target_compile_definitions(${basename} PRIVATE ${RUNTIME_DEFINES})
             else()
                 add_custom_command(OUTPUT "${LL}"
-                                   COMMAND ${CMAKE_C_COMPILER_LAUNCHER} $<TARGET_FILE:clang> ${clang_flags} -o "${LL}" "$<SHELL_PATH:${SOURCE}>"
+                                   COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} ${CMAKE_C_COMPILER_LAUNCHER} $<TARGET_FILE:clang> ${clang_flags} -o "${LL}" "$<SHELL_PATH:${SOURCE}>"
                                    DEPENDS "${SOURCE}"
                                    DEPFILE "${basename}.d"
                                    VERBATIM)
             endif()
 
             add_custom_command(OUTPUT "${BC}"
-                               COMMAND llvm-as "${LL}" -o "${BC}"
+                               COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:llvm-as> "${LL}" -o "${BC}"
                                DEPENDS "${LL}"
                                VERBATIM)
 
@@ -294,7 +294,7 @@ foreach (i IN LISTS RUNTIME_LL)
     endif ()
 
     add_custom_command(OUTPUT "${BC}"
-                       COMMAND llvm-as "${LL_TRANSFORMED}" -o "${BC}"
+                       COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:llvm-as> "${LL_TRANSFORMED}" -o "${BC}"
                        DEPENDS "${LL_TRANSFORMED}"
                        VERBATIM)
     add_custom_command(OUTPUT "${INITMOD}"

5. Build Halide

export PATH=/opt/riscv/bin:$PATH

cmake -DLLVM_DIR=$(realpath llvm-install/lib/cmake/llvm) \
      -DClang_DIR=$(realpath llvm-install/lib/cmake/clang) \
      -DCMAKE_BUILD_TYPE=Release \
      -DZLIB_INCLUDE_DIR=/usr/include \
      -DZLIB_LIBRARY=/usr/lib/riscv64-linux-gnu/libz.so \
      -DCMAKE_TOOLCHAIN_FILE=$(realpath riscv64.toolchain.cmake) \
      -DWITH_TESTS=OFF \
      -DWITH_TUTORIALS=OFF \
      -DWITH_PYTHON_BINDINGS=OFF \
      -S Halide -B halide-build

cmake --build halide-build -j$(nproc --all)
cmake --install halide-build --prefix halide-install

So my main question if LLVM supports RISC-V JIT at this moment (is that OK to use 15.0.2 or need switch to the main branch).

Also, I've tried add -DLLVM_TARGET_ARCH="riscv64" which is mentioned as required for JIT, However, the error is the same.

[llvmbot]

@llvm/issue-subscribers-backend-risc-v

[jrtc27]

The older RuntimeDyld isn't supported but the newer JITLink is, I believe

[dkurt]

Thanks, @jrtc27! Am I right that MCJIT always uses RTDyld so the solution should be migration from MCJIT to ORC JIT?

[dkurt]

track progress here: halide/Halide#7161