Skip to content

Commit 4d97206

Browse files
magicheng0816magicheng0816
authored
feat: add cc_api interface for llm inference. (#475)
1 parent d26a145 commit 4d97206

File tree

10 files changed

+1232
-34
lines changed

10 files changed

+1232
-34
lines changed

‎cmake/cc_shared_library.cmake‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
include(CMakeParseArguments)
2+
3+
# inspired by https://github.com/abseil/abseil-cpp
4+
# cc_shared_library()
5+
# CMake function to imitate Bazel's cc_shared_library rule.
6+
#
7+
# Parameters:
8+
# NAME: name of target
9+
# HDRS: List of public header files for the library
10+
# SRCS: List of source files for the library
11+
# DEPS: List of other libraries to be linked in to the binary targets
12+
# COPTS: List of private compile options
13+
# DEFINES: List of public defines
14+
# LINKOPTS: List of link options
15+
#
16+
# cc_library(
17+
# NAME
18+
# awesome
19+
# HDRS
20+
# "a.h"
21+
# SRCS
22+
# "a.cc"
23+
# )
24+
# cc_shared_library(
25+
# NAME
26+
# fantastic_lib
27+
# SRCS
28+
# "b.cc"
29+
# DEPS
30+
# :awesome
31+
# )
32+
#
33+
function(cc_shared_library)
34+
cmake_parse_arguments(
35+
CC_LIB # prefix
36+
"TESTONLY" # options
37+
"NAME" # one value args
38+
"HDRS;SRCS;COPTS;DEFINES;LINKOPTS;DEPS;INCLUDES" # multi value args
39+
${ARGN}
40+
)
41+
42+
if(CC_LIB_TESTONLY AND (NOT BUILD_TESTING))
43+
return()
44+
endif()
45+
46+
# Check if this is a header only library
47+
set(_CC_SRCS "${CC_LIB_SRCS}")
48+
foreach(src_file IN LISTS _CC_SRCS)
49+
if(${src_file} MATCHES ".*\\.(h|inc)")
50+
list(REMOVE_ITEM _CC_SRCS "${src_file}")
51+
endif()
52+
endforeach()
53+
54+
if(_CC_SRCS STREQUAL "")
55+
set(CC_LIB_IS_INTERFACE 1)
56+
else()
57+
set(CC_LIB_IS_INTERFACE 0)
58+
endif()
59+
60+
if(NOT CC_LIB_IS_INTERFACE)
61+
add_library(${CC_LIB_NAME} SHARED)
62+
target_sources(${CC_LIB_NAME}
63+
PRIVATE ${CC_LIB_SRCS} ${CC_LIB_HDRS})
64+
65+
target_compile_options(${CC_LIB_NAME} PRIVATE
66+
-fvisibility=hidden
67+
-fvisibility-inlines-hidden
68+
)
69+
70+
target_link_options(${CC_LIB_NAME} PRIVATE
71+
-Wl,--exclude-libs,ALL
72+
-Wl,--no-undefined
73+
)
74+
target_link_options(${CC_LIB_NAME} PRIVATE -Wl,--whole-archive)
75+
target_link_libraries(${CC_LIB_NAME}
76+
PUBLIC ${CC_LIB_DEPS}
77+
PRIVATE ${CC_LIB_LINKOPTS}
78+
)
79+
target_link_options(${CC_LIB_NAME} PRIVATE -Wl,--no-whole-archive)
80+
target_include_directories(${CC_LIB_NAME}
81+
PUBLIC
82+
"$<BUILD_INTERFACE:${COMMON_INCLUDE_DIRS}>"
83+
${CC_LIB_INCLUDES}
84+
)
85+
target_compile_options(${CC_LIB_NAME} PRIVATE ${CC_LIB_COPTS})
86+
target_compile_definitions(${CC_LIB_NAME} PUBLIC ${CC_LIB_DEFINES})
87+
else()
88+
# Generating header only library
89+
add_library(${CC_LIB_NAME} INTERFACE)
90+
target_include_directories(${CC_LIB_NAME}
91+
INTERFACE
92+
"$<BUILD_INTERFACE:${COMMON_INCLUDE_DIRS}>"
93+
${CC_LIB_INCLUDES}
94+
)
95+
96+
target_link_libraries(${CC_LIB_NAME}
97+
INTERFACE ${CC_LIB_DEPS} ${CC_LIB_LINKOPTS}
98+
)
99+
target_compile_definitions(${CC_LIB_NAME} INTERFACE ${CC_LIB_DEFINES})
100+
endif()
101+
102+
# add alias for the library target
103+
add_library(:${CC_LIB_NAME} ALIAS ${CC_LIB_NAME})
104+
endfunction()

‎setup.py‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ class ExtBuild(build_ext):
243243
("device=", None, "target device type (a3 or a2 or mlu or cuda)"),
244244
("arch=", None, "target arch type (x86 or arm)"),
245245
("install-xllm-kernels=", None, "install xllm_kernels RPM package (true/false)"),
246+
("generate-so=", None, "generate so or binary"),
246247
]
247248

248249
def initialize_options(self):
@@ -251,6 +252,7 @@ def initialize_options(self):
251252
self.device = None
252253
self.arch = None
253254
self.install_xllm_kernels = None
255+
self.generate_so = False
254256

255257
def finalize_options(self):
256258
build_ext.finalize_options(self)
@@ -329,6 +331,12 @@ def build_extension(self, ext: CMakeExtension):
329331
else:
330332
raise ValueError("Please set --device to a2 or a3 or mlu or cuda.")
331333

334+
product = "xllm"
335+
if self.generate_so:
336+
product = "libxllm.so"
337+
cmake_args += ["-DGENERATE_SO=ON"]
338+
else:
339+
cmake_args += ["-DGENERATE_SO=OFF"]
332340

333341
# Adding CMake arguments set as environment variable
334342
# (needed e.g. to build for ARM OSx on conda-forge)
@@ -361,7 +369,7 @@ def build_extension(self, ext: CMakeExtension):
361369

362370
os.makedirs(os.path.join(os.path.dirname(cmake_dir), "xllm/core/server/"), exist_ok=True)
363371
shutil.copy(
364-
os.path.join(extdir, "xllm"),
372+
os.path.join(extdir, product),
365373
os.path.join(os.path.dirname(cmake_dir), "xllm/core/server/"),
366374
)
367375

@@ -559,6 +567,7 @@ def pre_build():
559567
device = 'a2' # default
560568
arch = get_cpu_arch()
561569
install_kernels = True
570+
generate_so = False
562571
if '--device' in sys.argv:
563572
idx = sys.argv.index('--device')
564573
if idx + 1 < len(sys.argv):
@@ -588,6 +597,20 @@ def pre_build():
588597
sys.argv.pop(idx)
589598
sys.argv.pop(idx)
590599

600+
if '--generate-so' in sys.argv:
601+
idx = sys.argv.index('--generate-so')
602+
if idx + 1 < len(sys.argv):
603+
generate_so_val = sys.argv[idx+1].lower()
604+
if generate_so_val in ('true', '1', 'yes', 'y', 'on'):
605+
generate_so = True
606+
elif generate_so_val in ('false', '0', 'no', 'n', 'off'):
607+
generate_so = False
608+
else:
609+
print("Error: --generate-so must be true or false")
610+
sys.exit(1)
611+
sys.argv.pop(idx)
612+
sys.argv.pop(idx)
613+
591614
if "SKIP_TEST" in os.environ:
592615
BUILD_TEST_FILE = False
593616
if "SKIP_EXPORT" in os.environ:
@@ -635,7 +658,8 @@ def pre_build():
635658
options={'build_ext': {
636659
'device': device,
637660
'arch': arch,
638-
'install_xllm_kernels': install_kernels if install_kernels is not None else "false"
661+
'install_xllm_kernels': install_kernels if install_kernels is not None else "false",
662+
'generate_so': generate_so
639663
},
640664
'bdist_wheel': {
641665
'device': device,

‎xllm/CMakeLists.txt‎

Lines changed: 61 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
include(cc_binary)
2+
include(cc_shared_library)
23

34
include_directories(.)
45

6+
option(GENERATE_SO "Enable compile to generate .so" OFF)
7+
58
add_subdirectory(api_service)
69
add_subdirectory(core)
710
add_subdirectory(function_call)
@@ -12,22 +15,48 @@ add_subdirectory(proto)
1215
add_subdirectory(pybind)
1316
add_subdirectory(server)
1417

15-
cc_binary(
16-
NAME
17-
xllm
18-
SRCS
19-
xllm.cpp
20-
DEPS
21-
:flags
22-
:xllm_server
23-
:master
24-
absl::strings
25-
Boost::serialization
26-
gflags::gflags
27-
glog::glog
28-
Folly::folly
29-
nlohmann_json::nlohmann_json
30-
)
18+
if(GENERATE_SO)
19+
message(STATUS "build libxllm.so and install")
20+
cc_shared_library(
21+
NAME
22+
xllm
23+
HDRS
24+
cc_api/llm.h
25+
cc_api/types.h
26+
cc_api/internal.h
27+
SRCS
28+
cc_api/llm.cpp
29+
DEPS
30+
:flags
31+
:master
32+
absl::strings
33+
Boost::serialization
34+
gflags::gflags
35+
glog::glog
36+
Folly::folly
37+
nlohmann_json::nlohmann_json
38+
)
39+
40+
else()
41+
message(STATUS "build xllm binary and install")
42+
cc_binary(
43+
NAME
44+
xllm
45+
SRCS
46+
xllm.cpp
47+
DEPS
48+
:flags
49+
:xllm_server
50+
:master
51+
absl::strings
52+
Boost::serialization
53+
gflags::gflags
54+
glog::glog
55+
Folly::folly
56+
nlohmann_json::nlohmann_json
57+
)
58+
59+
endif()
3160

3261
# link brpc
3362
target_link_libraries(xllm PRIVATE glog::glog brpc leveldb::leveldb ZLIB::ZLIB protobuf::libprotobuf ${OpenCV_LIBS})
@@ -44,25 +73,25 @@ if(USE_MSPTI)
4473
endif()
4574
target_link_libraries(xllm PUBLIC ${COMMON_LIBS})
4675

47-
# install xllm binary
76+
# install xllm
4877
install(TARGETS xllm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
78+
4979
# Install all dependencies for xllm
5080
install(CODE [[
51-
file(GET_RUNTIME_DEPENDENCIES
52-
RESOLVED_DEPENDENCIES_VAR DEPENDENCIES
53-
UNRESOLVED_DEPENDENCIES_VAR UNRESOLVED_DEPENDENCIES
54-
EXECUTABLES $<TARGET_FILE:xllm>)
81+
file(GET_RUNTIME_DEPENDENCIES
82+
RESOLVED_DEPENDENCIES_VAR DEPENDENCIES
83+
UNRESOLVED_DEPENDENCIES_VAR UNRESOLVED_DEPENDENCIES
84+
EXECUTABLES $<TARGET_FILE:xllm>)
5585

56-
file(INSTALL
57-
DESTINATION "${CMAKE_INSTALL_PREFIX}/lib"
58-
FILES ${DEPENDENCIES}
59-
FOLLOW_SYMLINK_CHAIN)
86+
file(INSTALL
87+
DESTINATION "${CMAKE_INSTALL_PREFIX}/lib"
88+
FILES ${DEPENDENCIES}
89+
FOLLOW_SYMLINK_CHAIN)
6090

61-
# This should not be possible, but error out when a dependency cannot
62-
# be resolved.
63-
list(LENGTH UNRESOLVED_DEPENDENCIES UNRESOLVED_LENGTH)
64-
if(${UNRESOLVED_LENGTH} GREATER 0)
65-
message(FATAL_ERROR "Unresolved dependencies: ${UNRESOLVED_DEPENDENCIES}")
66-
endif()
91+
# This should not be possible, but error out when a dependency cannot
92+
# be resolved.
93+
list(LENGTH UNRESOLVED_DEPENDENCIES UNRESOLVED_LENGTH)
94+
if(${UNRESOLVED_LENGTH} GREATER 0)
95+
message(FATAL_ERROR "Unresolved dependencies: ${UNRESOLVED_DEPENDENCIES}")
96+
endif()
6797
]])
68-

0 commit comments

Comments
 (0)