diff --git a/Dockerfile.build b/Dockerfile.build index f967f69..990dcee 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -44,15 +44,6 @@ RUN rm -rf /tmp/flatbuffers # Install boost RUN dnf install -y boost boost-devel -# Install pybind11 for different versions of built-in python3 by almalinux -RUN /opt/python/cp310-cp310/bin/pip3 install pybind11 -RUN /opt/python/cp311-cp311/bin/pip3 install pybind11 - -# In almalinux, setuptools for python3.12 is not installed -# so install it -RUN /opt/python/cp312-cp312/bin/pip3 install setuptools -RUN /opt/python/cp312-cp312/bin/pip3 install pybind11 - # The above get the build environment ready! WORKDIR /app RUN git config --global --add safe.directory /app diff --git a/README.md b/README.md index 8a2a358..3a0849f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Most users just need to deploy and run InfiniStore, and they don't need to under pip install infinistore ``` -## Install from Source Code +## Development Installation For users who need to understand how InfiniStore code works or make code contributions to InfiniStore, it's recommended to install from source code: @@ -43,7 +43,7 @@ apt install libflatbuffers-dev apt install libspdlog-dev libfmt-dev apt install ibverbs-utils libibverbs-dev apt install libboost-dev libboost-stacktrace-dev -pip install -e . +pip install --no-build-isolation -e . pip install pre-commit pre-commit install ``` diff --git a/build_manylinux_wheels.sh b/build_manylinux_wheels.sh index 94511a1..8cca1a1 100644 --- a/build_manylinux_wheels.sh +++ b/build_manylinux_wheels.sh @@ -1,21 +1,19 @@ PYTHON_VERSIONS=( - "/opt/python/cp310-cp310/bin/python3.10" - "/opt/python/cp311-cp311/bin/python3.11" - "/opt/python/cp312-cp312/bin/python3.12" + "/opt/python/cp310-cp310/bin/python3.10" + "/opt/python/cp311-cp311/bin/python3.11" + "/opt/python/cp312-cp312/bin/python3.12" ) -rm -rf build/ dist/ wheelhouse/ + +#clean up inplace build +rm infinistore/*.so + +rm -rf dist/ wheelhouse/ +OLDPATH=$PATH for PYTHON in "${PYTHON_VERSIONS[@]}"; do - make -C src clean - rm -rf infinistore/*.so - make -C src manylinux PYTHON=${PYTHON} -j8 - if [ $? -ne 0 ]; then - exit 1 - fi - unset LD_LIBRARY_PATH - export LD_LIBRARY_PATH=/usr/local/lib - #ldd check, auditwheel will also check LD_LIBRARY_PATH - ldd src/*.so - ${PYTHON} setup.py bdist_wheel + BINDIR="$(dirname $PYTHON)" + export PATH="$BINDIR:$OLDPATH" + + pip wheel -v . --no-deps -w dist/ #runtime will install ibverbs, so exclude it WHEEL_FILE=$(ls dist/*.whl) echo "WHEEL_FILE: ${WHEEL_FILE}" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8c44630 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "meson", + "ninja", + "pybind11", +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py index b4ed2c0..a2a73f2 100644 --- a/setup.py +++ b/setup.py @@ -29,30 +29,44 @@ def get_version(): # invoke the make command to build the shared library class CustomBuildExt(build_ext): def run(self): + import glob + import shutil + import os + + subprocess.check_call(["meson", "setup", "build", "--wipe"], cwd="src") + subprocess.check_call(["ninja"], cwd="src/build") + + so_files = glob.glob("src/build/_infinistore*.so") if self.inplace: - # developer mode - print("developer mode: building shared library") - subprocess.check_call(["make", "clean"], cwd="src") - subprocess.check_call(["make"], cwd="src") - super().run() + for so_file in so_files: + dest = os.path.join("infinistore", os.path.basename(so_file)) + print(f"Copying {so_file} to {dest}") + shutil.copy(so_file, dest) else: - # package mode, return. build.sh script will build the shared library - return + build_dir = os.path.join(self.build_lib, "infinistore") + for so_file in so_files: + build_dest = os.path.join(build_dir, os.path.basename(so_file)) + print(f"Copying {so_file} to build directory: {build_dest}") + shutil.copy(so_file, build_dest) cpp_extension = Extension(name="infinistore._infinistore", sources=[]) ext_modules = [cpp_extension] - setup( name="infinistore", version=get_version(), packages=find_packages(), cmdclass={"build_ext": CustomBuildExt}, package_data={ - "infinistore": ["*.so"], + "infinistore": ["_infinistore*.so"], }, - install_requires=["torch", "uvloop", "fastapi", "pybind11", "uvicorn", "numpy"], + install_requires=[ + "uvloop", + "fastapi", + "uvicorn", + "numpy", + ], description="A kvcache memory pool", long_description=open("README.md").read(), long_description_content_type="text/markdown", diff --git a/src/Makefile b/src/Makefile deleted file mode 100644 index bb22870..0000000 --- a/src/Makefile +++ /dev/null @@ -1,44 +0,0 @@ -CXX = g++ -CXXFLAGS = -std=c++17 -Wall -O3 -g - -INCLUDES = -I/usr/local/ -LDFLAGS = -rdynamic -LIBS = -luv -libverbs -lfmt -lboost_stacktrace_basic -ldl -PYTHON=python3 -PYBIND11_INCLUDES = $(shell $(PYTHON) -m pybind11 --includes) -PYTHON_EXTENSION_SUFFIX = $(shell $(PYTHON)-config --extension-suffix) - -PYBIND_TARGET= _infinistore$(PYTHON_EXTENSION_SUFFIX) - -SOURCES := $(wildcard *.cpp) -OBJECTS = $(SOURCES:.cpp=.o) - -all:$(PYBIND_TARGET) - -manylinux: PYTHON ?= python3.11 -manylinux: CXXFLAGS = -std=c++17 -g -O3 -Wall -manylinux: LIBS = -luv -libverbs -lfmt -lboost_stacktrace_basic -ldl -manylinux: INCLUDES += -I/usr/local/include -manylinux: PYBIND11_INCLUDES = $(shell $(PYTHON) -m pybind11 --includes) -manylinux: PYTHON_EXTENSION_SUFFIX = $(shell $(PYTHON)-config --extension-suffix) -manylinux: PYBIND_TARGET = _infinistore$(PYTHON_EXTENSION_SUFFIX) -manylinux: $(PYBIND_TARGET) - -%.o: %.cpp - $(CXX) $(CXXFLAGS) $(INCLUDES) -MMD -MP -fPIC -c $< -o $@ - -%_generated.h: %.fbs - flatc --cpp $< - --include $(OBJECTS:.o=.d) - -$(PYBIND_TARGET): pybind.cpp libinfinistore.o utils.o protocol.o infinistore.o log.o ibv_helper.o mempool.o rdma.o - $(CXX) $(CXXFLAGS) $(INCLUDES) --shared -fPIC $(PYBIND11_INCLUDES) $^ \ - -o $(PYBIND_TARGET) $(LDFLAGS) $(LIBS) - rm -rf ../infinistore/$(PYBIND_TARGET) - cp $(PYBIND_TARGET) ../infinistore/ - - -.PHONY: clean test -clean: - rm -rf *.so *.o *.d test_client diff --git a/src/allocate_response_generated.h b/src/allocate_response_generated.h deleted file mode 100644 index 472d3c3..0000000 --- a/src/allocate_response_generated.h +++ /dev/null @@ -1,110 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -#ifndef FLATBUFFERS_GENERATED_ALLOCATERESPONSE_H_ -#define FLATBUFFERS_GENERATED_ALLOCATERESPONSE_H_ - -#include "flatbuffers/flatbuffers.h" - -struct RemoteBlock; - -struct RdmaAllocateResponse; -struct RdmaAllocateResponseBuilder; - -FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(8) RemoteBlock FLATBUFFERS_FINAL_CLASS { - private: - uint32_t rkey_; - int32_t padding0__; - uint64_t remote_addr_; - - public: - RemoteBlock() : rkey_(0), padding0__(0), remote_addr_(0) { (void)padding0__; } - RemoteBlock(uint32_t _rkey, uint64_t _remote_addr) - : rkey_(flatbuffers::EndianScalar(_rkey)), - padding0__(0), - remote_addr_(flatbuffers::EndianScalar(_remote_addr)) {} - uint32_t rkey() const { return flatbuffers::EndianScalar(rkey_); } - uint64_t remote_addr() const { return flatbuffers::EndianScalar(remote_addr_); } -}; -FLATBUFFERS_STRUCT_END(RemoteBlock, 16); - -struct RdmaAllocateResponse FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef RdmaAllocateResponseBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_BLOCKS = 4, - VT_ERROR_CODE = 6 - }; - const flatbuffers::Vector *blocks() const { - return GetPointer *>(VT_BLOCKS); - } - uint32_t error_code() const { return GetField(VT_ERROR_CODE, 0); } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_BLOCKS) && - verifier.VerifyVector(blocks()) && VerifyField(verifier, VT_ERROR_CODE) && - verifier.EndTable(); - } -}; - -struct RdmaAllocateResponseBuilder { - typedef RdmaAllocateResponse Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_blocks(flatbuffers::Offset> blocks) { - fbb_.AddOffset(RdmaAllocateResponse::VT_BLOCKS, blocks); - } - void add_error_code(uint32_t error_code) { - fbb_.AddElement(RdmaAllocateResponse::VT_ERROR_CODE, error_code, 0); - } - explicit RdmaAllocateResponseBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateRdmaAllocateResponse( - flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset> blocks = 0, - uint32_t error_code = 0) { - RdmaAllocateResponseBuilder builder_(_fbb); - builder_.add_error_code(error_code); - builder_.add_blocks(blocks); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateRdmaAllocateResponseDirect( - flatbuffers::FlatBufferBuilder &_fbb, const std::vector *blocks = nullptr, - uint32_t error_code = 0) { - auto blocks__ = blocks ? _fbb.CreateVectorOfStructs(*blocks) : 0; - return CreateRdmaAllocateResponse(_fbb, blocks__, error_code); -} - -inline const RdmaAllocateResponse *GetRdmaAllocateResponse(const void *buf) { - return flatbuffers::GetRoot(buf); -} - -inline const RdmaAllocateResponse *GetSizePrefixedRdmaAllocateResponse(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); -} - -inline bool VerifyRdmaAllocateResponseBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); -} - -inline bool VerifySizePrefixedRdmaAllocateResponseBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); -} - -inline void FinishRdmaAllocateResponseBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.Finish(root); -} - -inline void FinishSizePrefixedRdmaAllocateResponseBuffer( - flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset root) { - fbb.FinishSizePrefixed(root); -} - -#endif // FLATBUFFERS_GENERATED_ALLOCATERESPONSE_H_ diff --git a/src/delete_keys_generated.h b/src/delete_keys_generated.h deleted file mode 100644 index 3184cc9..0000000 --- a/src/delete_keys_generated.h +++ /dev/null @@ -1,84 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -#ifndef FLATBUFFERS_GENERATED_DELETEKEYS_H_ -#define FLATBUFFERS_GENERATED_DELETEKEYS_H_ - -#include "flatbuffers/flatbuffers.h" - -struct DeleteKeysRequest; -struct DeleteKeysRequestBuilder; - -struct DeleteKeysRequest FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef DeleteKeysRequestBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_KEYS = 4 }; - const flatbuffers::Vector> *keys() const { - return GetPointer> *>( - VT_KEYS); - } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_KEYS) && - verifier.VerifyVector(keys()) && verifier.VerifyVectorOfStrings(keys()) && - verifier.EndTable(); - } -}; - -struct DeleteKeysRequestBuilder { - typedef DeleteKeysRequest Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_keys( - flatbuffers::Offset>> keys) { - fbb_.AddOffset(DeleteKeysRequest::VT_KEYS, keys); - } - explicit DeleteKeysRequestBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateDeleteKeysRequest( - flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset>> keys = 0) { - DeleteKeysRequestBuilder builder_(_fbb); - builder_.add_keys(keys); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateDeleteKeysRequestDirect( - flatbuffers::FlatBufferBuilder &_fbb, - const std::vector> *keys = nullptr) { - auto keys__ = keys ? _fbb.CreateVector>(*keys) : 0; - return CreateDeleteKeysRequest(_fbb, keys__); -} - -inline const DeleteKeysRequest *GetDeleteKeysRequest(const void *buf) { - return flatbuffers::GetRoot(buf); -} - -inline const DeleteKeysRequest *GetSizePrefixedDeleteKeysRequest(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); -} - -inline bool VerifyDeleteKeysRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); -} - -inline bool VerifySizePrefixedDeleteKeysRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); -} - -inline void FinishDeleteKeysRequestBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.Finish(root); -} - -inline void FinishSizePrefixedDeleteKeysRequestBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.FinishSizePrefixed(root); -} - -#endif // FLATBUFFERS_GENERATED_DELETEKEYS_H_ diff --git a/src/get_match_last_index_generated.h b/src/get_match_last_index_generated.h deleted file mode 100644 index 1aeee7a..0000000 --- a/src/get_match_last_index_generated.h +++ /dev/null @@ -1,84 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -#ifndef FLATBUFFERS_GENERATED_GETMATCHLASTINDEX_H_ -#define FLATBUFFERS_GENERATED_GETMATCHLASTINDEX_H_ - -#include "flatbuffers/flatbuffers.h" - -struct GetMatchLastIndexRequest; -struct GetMatchLastIndexRequestBuilder; - -struct GetMatchLastIndexRequest FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef GetMatchLastIndexRequestBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_KEYS = 4 }; - const flatbuffers::Vector> *keys() const { - return GetPointer> *>( - VT_KEYS); - } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_KEYS) && - verifier.VerifyVector(keys()) && verifier.VerifyVectorOfStrings(keys()) && - verifier.EndTable(); - } -}; - -struct GetMatchLastIndexRequestBuilder { - typedef GetMatchLastIndexRequest Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_keys( - flatbuffers::Offset>> keys) { - fbb_.AddOffset(GetMatchLastIndexRequest::VT_KEYS, keys); - } - explicit GetMatchLastIndexRequestBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateGetMatchLastIndexRequest( - flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset>> keys = 0) { - GetMatchLastIndexRequestBuilder builder_(_fbb); - builder_.add_keys(keys); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateGetMatchLastIndexRequestDirect( - flatbuffers::FlatBufferBuilder &_fbb, - const std::vector> *keys = nullptr) { - auto keys__ = keys ? _fbb.CreateVector>(*keys) : 0; - return CreateGetMatchLastIndexRequest(_fbb, keys__); -} - -inline const GetMatchLastIndexRequest *GetGetMatchLastIndexRequest(const void *buf) { - return flatbuffers::GetRoot(buf); -} - -inline const GetMatchLastIndexRequest *GetSizePrefixedGetMatchLastIndexRequest(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); -} - -inline bool VerifyGetMatchLastIndexRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); -} - -inline bool VerifySizePrefixedGetMatchLastIndexRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); -} - -inline void FinishGetMatchLastIndexRequestBuffer( - flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset root) { - fbb.Finish(root); -} - -inline void FinishSizePrefixedGetMatchLastIndexRequestBuffer( - flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset root) { - fbb.FinishSizePrefixed(root); -} - -#endif // FLATBUFFERS_GENERATED_GETMATCHLASTINDEX_H_ diff --git a/src/local_meta_generated.h b/src/local_meta_generated.h deleted file mode 100644 index 8840cd9..0000000 --- a/src/local_meta_generated.h +++ /dev/null @@ -1,153 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -#ifndef FLATBUFFERS_GENERATED_LOCALMETA_H_ -#define FLATBUFFERS_GENERATED_LOCALMETA_H_ - -#include "flatbuffers/flatbuffers.h" - -struct Block; -struct BlockBuilder; - -struct LocalMeta; -struct LocalMetaBuilder; - -struct Block FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef BlockBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_KEY = 4, VT_OFFSET = 6 }; - const flatbuffers::String *key() const { - return GetPointer(VT_KEY); - } - uint64_t offset() const { return GetField(VT_OFFSET, 0); } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_KEY) && - verifier.VerifyString(key()) && VerifyField(verifier, VT_OFFSET) && - verifier.EndTable(); - } -}; - -struct BlockBuilder { - typedef Block Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_key(flatbuffers::Offset key) { - fbb_.AddOffset(Block::VT_KEY, key); - } - void add_offset(uint64_t offset) { fbb_.AddElement(Block::VT_OFFSET, offset, 0); } - explicit BlockBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateBlock(flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset key = 0, - uint64_t offset = 0) { - BlockBuilder builder_(_fbb); - builder_.add_offset(offset); - builder_.add_key(key); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateBlockDirect(flatbuffers::FlatBufferBuilder &_fbb, - const char *key = nullptr, - uint64_t offset = 0) { - auto key__ = key ? _fbb.CreateString(key) : 0; - return CreateBlock(_fbb, key__, offset); -} - -struct LocalMeta FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef LocalMetaBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_IPC_HANDLE = 4, - VT_BLOCK_SIZE = 6, - VT_BLOCKS = 8 - }; - const flatbuffers::Vector *ipc_handle() const { - return GetPointer *>(VT_IPC_HANDLE); - } - int32_t block_size() const { return GetField(VT_BLOCK_SIZE, 0); } - const flatbuffers::Vector> *blocks() const { - return GetPointer> *>(VT_BLOCKS); - } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_IPC_HANDLE) && - verifier.VerifyVector(ipc_handle()) && - VerifyField(verifier, VT_BLOCK_SIZE) && VerifyOffset(verifier, VT_BLOCKS) && - verifier.VerifyVector(blocks()) && verifier.VerifyVectorOfTables(blocks()) && - verifier.EndTable(); - } -}; - -struct LocalMetaBuilder { - typedef LocalMeta Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_ipc_handle(flatbuffers::Offset> ipc_handle) { - fbb_.AddOffset(LocalMeta::VT_IPC_HANDLE, ipc_handle); - } - void add_block_size(int32_t block_size) { - fbb_.AddElement(LocalMeta::VT_BLOCK_SIZE, block_size, 0); - } - void add_blocks(flatbuffers::Offset>> blocks) { - fbb_.AddOffset(LocalMeta::VT_BLOCKS, blocks); - } - explicit LocalMetaBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateLocalMeta( - flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset> ipc_handle = 0, int32_t block_size = 0, - flatbuffers::Offset>> blocks = 0) { - LocalMetaBuilder builder_(_fbb); - builder_.add_blocks(blocks); - builder_.add_block_size(block_size); - builder_.add_ipc_handle(ipc_handle); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateLocalMetaDirect( - flatbuffers::FlatBufferBuilder &_fbb, const std::vector *ipc_handle = nullptr, - int32_t block_size = 0, const std::vector> *blocks = nullptr) { - auto ipc_handle__ = ipc_handle ? _fbb.CreateVector(*ipc_handle) : 0; - auto blocks__ = blocks ? _fbb.CreateVector>(*blocks) : 0; - return CreateLocalMeta(_fbb, ipc_handle__, block_size, blocks__); -} - -inline const LocalMeta *GetLocalMeta(const void *buf) { - return flatbuffers::GetRoot(buf); -} - -inline const LocalMeta *GetSizePrefixedLocalMeta(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); -} - -inline bool VerifyLocalMetaBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); -} - -inline bool VerifySizePrefixedLocalMetaBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); -} - -inline void FinishLocalMetaBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.Finish(root); -} - -inline void FinishSizePrefixedLocalMetaBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.FinishSizePrefixed(root); -} - -#endif // FLATBUFFERS_GENERATED_LOCALMETA_H_ diff --git a/src/meson.build b/src/meson.build new file mode 100644 index 0000000..490ed02 --- /dev/null +++ b/src/meson.build @@ -0,0 +1,70 @@ +project('infinistore', +['cpp'], +version: '0.0.0', +default_options: [ + 'buildtype=release', + 'cpp_std=c++17', + 'warning_level=2', # -Wall + 'optimization=3', # -O3 + 'debug=true' # -g +]) + +python3 = import('python').find_installation('python3', pure: false) + +# C/C++ compiler interface +cc = meson.get_compiler('cpp') + +fb_sources = [ + 'meta_request.fbs', + 'allocate_response.fbs', + 'delete_keys.fbs', + 'get_match_last_index.fbs', + 'tcp_payload_request.fbs' +] + +flatc = find_program('flatc', required: true) + +generated_headers = [] + +foreach fb_source : fb_sources + fb_name = fb_source.split('.')[0] + header_name = fb_name + '_generated.h' + generated_headers += custom_target( + fb_name + '_gen', + input: fb_source, + output: header_name, + command: [ + flatc, + '--cpp', + '@INPUT@', + ], + ) +endforeach + +# External library dependencies +pybind_dep = dependency('pybind11', required: true) +libuv_dep = dependency('libuv', required: true) +fmt_dep = dependency('fmt', required: true) +boost_stack_dep = dependency('boost', modules: ['stacktrace_basic'], required: true) +ibverbs_dep = dependency('libibverbs', required: true) + +# Source files for the extension +ext_sources = [ + 'libinfinistore.cpp', + 'utils.cpp', + 'protocol.cpp', + 'infinistore.cpp', + 'log.cpp', + 'ibv_helper.cpp', + 'mempool.cpp', + 'rdma.cpp', + 'pybind.cpp' +] + + +# Build the Python extension module +python3.extension_module('_infinistore', + sources: ext_sources + generated_headers, + dependencies: [pybind_dep, libuv_dep, fmt_dep, boost_stack_dep, ibverbs_dep], + install: true +) \ No newline at end of file diff --git a/src/meta_request.fbs b/src/meta_request.fbs index b0b538b..dd8a756 100644 --- a/src/meta_request.fbs +++ b/src/meta_request.fbs @@ -6,4 +6,5 @@ table RemoteMetaRequest { remote_addrs: [ulong]; // GPU addresses op: byte; } + root_type RemoteMetaRequest; diff --git a/src/meta_request_generated.h b/src/meta_request_generated.h deleted file mode 100644 index db02580..0000000 --- a/src/meta_request_generated.h +++ /dev/null @@ -1,116 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -#ifndef FLATBUFFERS_GENERATED_METAREQUEST_H_ -#define FLATBUFFERS_GENERATED_METAREQUEST_H_ - -#include "flatbuffers/flatbuffers.h" - -struct RemoteMetaRequest; -struct RemoteMetaRequestBuilder; - -struct RemoteMetaRequest FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef RemoteMetaRequestBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_KEYS = 4, - VT_BLOCK_SIZE = 6, - VT_RKEY = 8, - VT_REMOTE_ADDRS = 10, - VT_OP = 12 - }; - const flatbuffers::Vector> *keys() const { - return GetPointer> *>( - VT_KEYS); - } - int32_t block_size() const { return GetField(VT_BLOCK_SIZE, 0); } - uint32_t rkey() const { return GetField(VT_RKEY, 0); } - const flatbuffers::Vector *remote_addrs() const { - return GetPointer *>(VT_REMOTE_ADDRS); - } - int8_t op() const { return GetField(VT_OP, 0); } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_KEYS) && - verifier.VerifyVector(keys()) && verifier.VerifyVectorOfStrings(keys()) && - VerifyField(verifier, VT_BLOCK_SIZE) && - VerifyField(verifier, VT_RKEY) && - VerifyOffset(verifier, VT_REMOTE_ADDRS) && verifier.VerifyVector(remote_addrs()) && - VerifyField(verifier, VT_OP) && verifier.EndTable(); - } -}; - -struct RemoteMetaRequestBuilder { - typedef RemoteMetaRequest Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_keys( - flatbuffers::Offset>> keys) { - fbb_.AddOffset(RemoteMetaRequest::VT_KEYS, keys); - } - void add_block_size(int32_t block_size) { - fbb_.AddElement(RemoteMetaRequest::VT_BLOCK_SIZE, block_size, 0); - } - void add_rkey(uint32_t rkey) { fbb_.AddElement(RemoteMetaRequest::VT_RKEY, rkey, 0); } - void add_remote_addrs(flatbuffers::Offset> remote_addrs) { - fbb_.AddOffset(RemoteMetaRequest::VT_REMOTE_ADDRS, remote_addrs); - } - void add_op(int8_t op) { fbb_.AddElement(RemoteMetaRequest::VT_OP, op, 0); } - explicit RemoteMetaRequestBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateRemoteMetaRequest( - flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset>> keys = 0, - int32_t block_size = 0, uint32_t rkey = 0, - flatbuffers::Offset> remote_addrs = 0, int8_t op = 0) { - RemoteMetaRequestBuilder builder_(_fbb); - builder_.add_remote_addrs(remote_addrs); - builder_.add_rkey(rkey); - builder_.add_block_size(block_size); - builder_.add_keys(keys); - builder_.add_op(op); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateRemoteMetaRequestDirect( - flatbuffers::FlatBufferBuilder &_fbb, - const std::vector> *keys = nullptr, - int32_t block_size = 0, uint32_t rkey = 0, const std::vector *remote_addrs = nullptr, - int8_t op = 0) { - auto keys__ = keys ? _fbb.CreateVector>(*keys) : 0; - auto remote_addrs__ = remote_addrs ? _fbb.CreateVector(*remote_addrs) : 0; - return CreateRemoteMetaRequest(_fbb, keys__, block_size, rkey, remote_addrs__, op); -} - -inline const RemoteMetaRequest *GetRemoteMetaRequest(const void *buf) { - return flatbuffers::GetRoot(buf); -} - -inline const RemoteMetaRequest *GetSizePrefixedRemoteMetaRequest(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); -} - -inline bool VerifyRemoteMetaRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); -} - -inline bool VerifySizePrefixedRemoteMetaRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); -} - -inline void FinishRemoteMetaRequestBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.Finish(root); -} - -inline void FinishSizePrefixedRemoteMetaRequestBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.FinishSizePrefixed(root); -} - -#endif // FLATBUFFERS_GENERATED_METAREQUEST_H_ diff --git a/src/tcp_payload_request_generated.h b/src/tcp_payload_request_generated.h deleted file mode 100644 index afe3162..0000000 --- a/src/tcp_payload_request_generated.h +++ /dev/null @@ -1,94 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify - -#ifndef FLATBUFFERS_GENERATED_TCPPAYLOADREQUEST_H_ -#define FLATBUFFERS_GENERATED_TCPPAYLOADREQUEST_H_ - -#include "flatbuffers/flatbuffers.h" - -struct TCPPayloadRequest; -struct TCPPayloadRequestBuilder; - -struct TCPPayloadRequest FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef TCPPayloadRequestBuilder Builder; - enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_KEY = 4, - VT_VALUE_LENGTH = 6, - VT_OP = 8 - }; - const flatbuffers::String *key() const { - return GetPointer(VT_KEY); - } - int32_t value_length() const { return GetField(VT_VALUE_LENGTH, 0); } - int8_t op() const { return GetField(VT_OP, 0); } - bool Verify(flatbuffers::Verifier &verifier) const { - return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_KEY) && - verifier.VerifyString(key()) && VerifyField(verifier, VT_VALUE_LENGTH) && - VerifyField(verifier, VT_OP) && verifier.EndTable(); - } -}; - -struct TCPPayloadRequestBuilder { - typedef TCPPayloadRequest Table; - flatbuffers::FlatBufferBuilder &fbb_; - flatbuffers::uoffset_t start_; - void add_key(flatbuffers::Offset key) { - fbb_.AddOffset(TCPPayloadRequest::VT_KEY, key); - } - void add_value_length(int32_t value_length) { - fbb_.AddElement(TCPPayloadRequest::VT_VALUE_LENGTH, value_length, 0); - } - void add_op(int8_t op) { fbb_.AddElement(TCPPayloadRequest::VT_OP, op, 0); } - explicit TCPPayloadRequestBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { - start_ = fbb_.StartTable(); - } - flatbuffers::Offset Finish() { - const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); - return o; - } -}; - -inline flatbuffers::Offset CreateTCPPayloadRequest( - flatbuffers::FlatBufferBuilder &_fbb, flatbuffers::Offset key = 0, - int32_t value_length = 0, int8_t op = 0) { - TCPPayloadRequestBuilder builder_(_fbb); - builder_.add_value_length(value_length); - builder_.add_key(key); - builder_.add_op(op); - return builder_.Finish(); -} - -inline flatbuffers::Offset CreateTCPPayloadRequestDirect( - flatbuffers::FlatBufferBuilder &_fbb, const char *key = nullptr, int32_t value_length = 0, - int8_t op = 0) { - auto key__ = key ? _fbb.CreateString(key) : 0; - return CreateTCPPayloadRequest(_fbb, key__, value_length, op); -} - -inline const TCPPayloadRequest *GetTCPPayloadRequest(const void *buf) { - return flatbuffers::GetRoot(buf); -} - -inline const TCPPayloadRequest *GetSizePrefixedTCPPayloadRequest(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); -} - -inline bool VerifyTCPPayloadRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); -} - -inline bool VerifySizePrefixedTCPPayloadRequestBuffer(flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); -} - -inline void FinishTCPPayloadRequestBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.Finish(root); -} - -inline void FinishSizePrefixedTCPPayloadRequestBuffer(flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { - fbb.FinishSizePrefixed(root); -} - -#endif // FLATBUFFERS_GENERATED_TCPPAYLOADREQUEST_H_