Add support for generating SHARED library with -DBUILD_SHARED_LIBS=ON flag

Author: PARTH SHAH

CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.19...3.30)
 project(tokenizers_cpp C CXX)
 
+option(BUILD_SHARED_LIBS "Build tokenizers_cpp as a shared library" OFF)
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -157,7 +159,12 @@ set(
   src/huggingface_tokenizer.cc
   src/rwkv_world_tokenizer.cc
 )
-add_library(tokenizers_cpp STATIC ${TOKENIZER_CPP_SRCS})
+
+if(BUILD_SHARED_LIBS)
+  add_library(tokenizers_cpp SHARED ${TOKENIZER_CPP_SRCS} src/static_exports.cc)
+else()
+  add_library(tokenizers_cpp STATIC ${TOKENIZER_CPP_SRCS})
+endif()
 target_include_directories(tokenizers_cpp PRIVATE sentencepiece/src)
 target_include_directories(tokenizers_cpp PRIVATE msgpack/include)
 target_include_directories(tokenizers_cpp PUBLIC ${TOKENIZERS_CPP_INCLUDE})

src/static_exports.cc

@@ -0,0 +1,65 @@
+#include "tokenizers_cpp.h"
+
+// export LoadBlobJsonAndEncode(const std::string &, const std::string &, std::vector<int32_t> &)
+extern "C" __declspec(dllexport) void LoadBlobJsonAndEncode(const std::string& json_blob, const std::string& text, std::vector<int32_t>& token_ids)
+{
+  auto tokenizer = tokenizers::Tokenizer::FromBlobJSON(json_blob);
+  token_ids = tokenizer->Encode(text);
+  return;
+}
+
+// export LoadBlobJsonAndEncodeBatch(const std::string &, const std::vector<std::string> &, std::vector<std::vector<int32_t>> &)
+extern "C" __declspec(dllexport) void LoadBlobJsonAndEncodeBatch(const std::string& json_blob, const std::vector<std::string>& texts, std::vector<std::vector<int32_t>>& token_ids_batch)
+{
+  auto tokenizer = tokenizers::Tokenizer::FromBlobJSON(json_blob);
+  token_ids_batch = tokenizer->EncodeBatch(texts);
+  return;
+}
+
+// export LoadBlobSentencePieceAndEncode(const std::string &, const std::string &, std::vector<int32_t> &)
+extern "C" __declspec(dllexport) void LoadBlobSentencePieceAndEncode(const std::string& model_blob, const std::string& text, std::vector<int32_t>& token_ids)
+{
+    auto tokenizer = tokenizers::Tokenizer::FromBlobSentencePiece(model_blob);
+    token_ids = tokenizer->Encode(text);
+    return;
+}
+
+// export LoadBlobSentencePieceAndEncodeBatch(const std::string &, const std::vector<std::string> &, std::vector<std::vector<int32_t>> &)
+extern "C" __declspec(dllexport) void LoadBlobSentencePieceAndEncodeBatch(const std::string& model_blob, const std::vector<std::string>& texts, std::vector<std::vector<int32_t>>& token_ids_batch)
+{
+    auto tokenizer = tokenizers::Tokenizer::FromBlobSentencePiece(model_blob);
+    token_ids_batch = tokenizer->EncodeBatch(texts);
+    return;
+}
+
+// export LoadBlobRWKVWorldAndEncode(const std::string &, const std::string &, std::vector<int32_t> &)
+extern "C" __declspec(dllexport) void LoadBlobRWKVWorldAndEncode(const std::string& model_blob, const std::string& text, std::vector<int32_t>& token_ids)
+{
+    auto tokenizer = tokenizers::Tokenizer::FromBlobRWKVWorld(model_blob);
+    token_ids = tokenizer->Encode(text);
+    return;
+}
+
+// export LoadBlobRWKVWorldAndEncodeBatch(const std::string &, const std::vector<std::string> &, std::vector<std::vector<int32_t>> &)
+extern "C" __declspec(dllexport) void LoadBlobRWKVWorldAndEncodeBatch(const std::string& model_blob, const std::vector<std::string>& texts, std::vector<std::vector<int32_t>>& token_ids_batch)
+{
+    auto tokenizer = tokenizers::Tokenizer::FromBlobRWKVWorld(model_blob);
+    token_ids_batch = tokenizer->EncodeBatch(texts);
+    return;
+}
+
+// export LoadBlobByteLevelBPEAndEncode(const std::string &, const std::string &, const std::string &, const std::string &, std::vector<int32_t> &)
+extern "C" __declspec(dllexport) void LoadBlobByteLevelBPEAndEncode(const std::string& vocab_blob, const std::string& merges_blob, const std::string& added_tokens, const std::string& text, std::vector<int32_t>& token_ids)
+{
+    auto tokenizer = tokenizers::Tokenizer::FromBlobByteLevelBPE(vocab_blob, merges_blob, added_tokens);
+    token_ids = tokenizer->Encode(text);
+    return;
+}
+
+// export LoadBlobByteLevelBPEAndEncodeBatch(const std::string &, const std::string &, const std::string &, const std::vector<std::string> &, std::vector<std::vector<int32_t>> &)
+extern "C" __declspec(dllexport) void LoadBlobByteLevelBPEAndEncodeBatch(const std::string& vocab_blob, const std::string& merges_blob, const std::string& added_tokens, const std::vector<std::string>& texts, std::vector<std::vector<int32_t>>& token_ids_batch)
+{
+    auto tokenizer = tokenizers::Tokenizer::FromBlobByteLevelBPE(vocab_blob, merges_blob, added_tokens);
+    token_ids_batch = tokenizer->EncodeBatch(texts);
+    return;
+}