feat: support iluvatar backend qwen3 0.6b run through

Author: zuolve.lai

CMakeLists.txt

@@ -3,6 +3,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
 option(USE_NPU "Enable NPU support" OFF)
 option(USE_MLU "Enable MLU support" OFF)
+option(USE_ILU "Enable ILU support" OFF)
 option(USE_CUDA "Enable CUDA support" OFF)
 add_compile_definitions(YLT_ENABLE_IBV)
 add_definitions(-DYLT_ENABLE_IBV)
@@ -105,7 +106,7 @@ set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS ON)
 
-if(USE_NPU OR USE_CUDA)
+if(USE_NPU OR USE_CUDA OR USE_ILU)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
   add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
 elseif(USE_MLU)
@@ -208,6 +209,19 @@ if(USE_CUDA)
   message(STATUS "TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}")
 endif()
 
+if(USE_ILU)
+    set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
+    set(CMAKE_EXPORT_COMPILE_COMMANDS   ON)
+    set(CMAKE_CUDA_ARCHITECTURES "ivcore11")
+    set(WARNINGS_AS_ERRORS OFF)
+    if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+      add_definitions(
+        -Wno-c++11-narrowing
+        -Wno-thread-safety-analysis
+      )
+  endif()
+endif()
+
 # configure vcpkg
 # have to set CMAKE_TOOLCHAIN_FILE before first project call.
 # if (DEFINED ENV{VCPKG_ROOT} AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
@@ -424,6 +438,23 @@ if(USE_CUDA)
   )
 endif()
 
+if(USE_ILU)
+  add_definitions(-DUSE_ILU)
+  set(CMAKE_VERBOSE_MAKEFILE ON)
+  include_directories(
+      $ENV{PYTHON_INCLUDE_PATH}
+      $ENV{PYTORCH_INSTALL_PATH}/include
+      $ENV{PYTORCH_INSTALL_PATH}/include/torch/csrc/api/include
+      $ENV{IXFORMER_INSTALL_PATH}/csrc/include/ixformer
+  )
+
+  link_directories(
+    $ENV{PYTHON_LIB_PATH}
+    $ENV{PYTORCH_INSTALL_PATH}/lib
+    $ENV{IXFORMER_INSTALL_PATH}
+  )
+endif()
+
 # check if USE_CXX11_ABI is set correctly
 # if (DEFINED USE_CXX11_ABI)
 #   parse_make_options(${TORCH_CXX_FLAGS} "TORCH_CXX_FLAGS")

setup.py

@@ -106,11 +106,11 @@ def get_torch_root_path():
     except ImportError:
         return None
 
-def get_torch_mlu_root_path():
+def get_ixformer_root_path():
     try:
-        import torch_mlu
+        import ixformer
         import os
-        return os.path.dirname(os.path.abspath(torch_mlu.__file__))
+        return os.path.dirname(os.path.abspath(ixformer.__file__))
     except ImportError:
         return None
 
@@ -226,7 +226,15 @@ def set_cuda_envs():
     os.environ["CUDA_TOOLKIT_ROOT_DIR"] = "/usr/local/cuda"
     os.environ["NCCL_ROOT"] = get_nccl_root_path()
     os.environ["NCCL_VERSION"] = "2"
-    
+
+def set_ilu_envs():
+    os.environ["PYTHON_INCLUDE_PATH"] = get_python_include_path()
+    os.environ["PYTHON_LIB_PATH"] =  get_torch_root_path()
+    os.environ["LIBTORCH_ROOT"] = get_torch_root_path()
+    os.environ["PYTORCH_INSTALL_PATH"] = get_torch_root_path()
+    os.environ["CUDA_TOOLKIT_ROOT_DIR"] = "/usr/local/corex"
+    os.environ["IXFORMER_INSTALL_PATH"] = get_ixformer_root_path()
+
 class CMakeExtension(Extension):
     def __init__(self, name: str, path: str, sourcedir: str = "") -> None:
         super().__init__(name, sources=[])
@@ -275,8 +283,7 @@ def run(self):
             for ext in self.extensions:
                 self.build_extension(ext)
         except Exception as e:
-            print("ERROR: Build failed.")
-            print(f"Details: {e}")
+            print("Build failed.")
             exit(1)
 
     def build_extension(self, ext: CMakeExtension):
@@ -308,7 +315,7 @@ def build_extension(self, ext: CMakeExtension):
             f"-DDEVICE_ARCH={self.arch.upper()}",
             f"-DINSTALL_XLLM_KERNELS={'ON' if self.install_xllm_kernels else 'OFF'}",
         ]
-        
+
         if self.device == "a2" or self.device == "a3":
             cmake_args += ["-DUSE_NPU=ON"]
             # set npu environment variables
@@ -323,6 +330,9 @@ def build_extension(self, ext: CMakeExtension):
                            f"-DCMAKE_CUDA_ARCHITECTURES={cuda_architectures}"]
             # set cuda environment variables
             set_cuda_envs()
+        elif self.device == "ilu":
+            cmake_args += ["-DUSE_ILU=ON"]
+            set_ilu_envs()
         else:
             raise ValueError("Please set --device to a2 or a3 or mlu or cuda.")
 
@@ -340,6 +350,7 @@ def build_extension(self, ext: CMakeExtension):
 
         build_args = ["--config", build_type]
         max_jobs = os.getenv("MAX_JOBS", str(os.cpu_count()))
+        # max_jobs="2"
         build_args += ["-j" + max_jobs]
 
         env = os.environ.copy()
@@ -553,7 +564,7 @@ def pre_build():
             exit(0)
 
 if __name__ == "__main__":
-    device = 'a2'  # default
+    device = 'ilu'  # default
     arch = get_cpu_arch()
     install_kernels = True
     if '--device' in sys.argv:

third_party/CMakeLists.txt

@@ -19,4 +19,32 @@ target_include_directories(mooncake_store PUBLIC
   ${CMAKE_CURRENT_SOURCE_DIR}/Mooncake/mooncake-transfer-engine/include
 )
 
+if(USE_ILU)
+  if(TARGET cpprest)
+      set_target_properties(cpprest PROPERTIES
+          CXX_STANDARD 20
+          CXX_STANDARD_REQUIRED ON
+          CXX_EXTENSIONS OFF
+      )
+  endif()
+  if(TARGET transfer_engine)
+    target_compile_options(transfer_engine PRIVATE -std=c++20)
+    set_target_properties(transfer_engine PROPERTIES
+        CXX_STANDARD 20
+        CXX_STANDARD_REQUIRED ON
+    )
+    message(STATUS "Set C++20 for transfer_engine target")
+  endif()
+  if(TARGET SMHasherSupport)
+      set_target_properties(SMHasherSupport PROPERTIES
+          CXX_STANDARD 11
+          CXX_STANDARD_REQUIRED ON
+          CXX_EXTENSIONS OFF
+      )
+      message(STATUS "SMHasherSupport target found and configured")
+  else()
+      message(WARNING "SMHasherSupport target not found after adding smhasher")
+  endif()
+endif()
+
 target_link_libraries(mooncake_store PUBLIC transfer_engine cachelib_memory_allocator)

third_party/brpc

@@ -1,8 +1,18 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 047ae3e..b4ae201 100644
+index 047ae3e..634b1bf 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -16,7 +16,7 @@ option(WITH_STORE "build mooncake store library and sample code" ON)
+@@ -1,6 +1,8 @@
+ cmake_minimum_required(VERSION 3.16)
+ project(mooncake CXX C)
+-
++set(CMAKE_CXX_STANDARD 20)
++set(CMAKE_CXX_STANDARD_REQUIRED ON)
++set(CMAKE_CXX_EXTENSIONS OFF)
+ # indicates cmake is invoked from top-level dir
+ set(GLOBAL_CONFIG "true")
+ 
+@@ -16,7 +18,7 @@ option(WITH_STORE "build mooncake store library and sample code" ON)
  option(WITH_P2P_STORE "build p2p store library and sample code" OFF)
  option(WITH_RUST_EXAMPLE "build the Rust interface and sample code for the transfer engine" OFF)
 

third_party/custom_patch/Mooncake.patch

@@ -104,7 +104,7 @@ void WorkerServer::create_server(
 
   CollectiveCommunicator comm(worker_global_rank, world_size, dp_size, ep_size);
   const ParallelArgs* parallel_args = comm.parallel_args();
-#if defined(USE_MLU) || defined(USE_CUDA)
+#if defined(USE_MLU) || defined(USE_CUDA) || defined(USE_ILU)
   comm.create_process_groups(master_node_addr, device);
 #endif
 

third_party/hccl_transfer

@@ -196,7 +196,7 @@ void BatchInputBuilder::process_sequences_multithreaded() {
     state_.q_seq_lens.insert(state_.q_seq_lens.end(),
                              state.q_seq_lens.begin(),
                              state.q_seq_lens.end());
-#elif defined(USE_MLU) || defined(USE_CUDA)
+#elif defined(USE_MLU) || defined(USE_CUDA) || defined(USE_ILU)
     int32_t seq_len_offset = state_.seq_lens.back();
     // skip the first element which is 0
     for (size_t i = 1; i < state.seq_lens.size(); ++i) {
@@ -281,7 +281,7 @@ void BatchInputBuilder::process_single_sequence(
 #if defined(USE_NPU)
   state.seq_lens.push_back(seq_len);
   state.q_seq_lens.push_back(q_seq_len);
-#elif defined(USE_MLU) || defined(USE_CUDA)
+#elif defined(USE_MLU) || defined(USE_CUDA) || defined(USE_ILU)
   state.seq_lens.push_back(state.seq_lens.back() + seq_len);
   state.q_seq_lens.push_back(state.q_seq_lens.back() + q_seq_len);
 #endif
@@ -510,7 +510,7 @@ void BatchInputBuilder::padding_decode_batch_size(
 #if defined(USE_NPU)
         state_.seq_lens.push_back(num_decoding_tokens);
         state_.q_seq_lens.push_back(num_decoding_tokens);
-#elif defined(USE_MLU) || defined(USE_CUDA)
+#elif defined(USE_MLU) || defined(USE_CUDA) || defined(USE_ILU)
         state_.seq_lens.push_back(state_.seq_lens.back() + num_decoding_tokens);
         state_.q_seq_lens.push_back(state_.q_seq_lens.back() +
                                     num_decoding_tokens);