feat: Add Protobuf log handler, improve CI toolchain, and enhance code generation

.github/workflows/testing-cpp.yml

@@ -18,25 +18,30 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
+            go-version: 1.21.x
+            init_script: bash init.sh
             gen_script: bash gen.sh
-            cmake_config: cmake -S . -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Debug
             run_loader: ./bin/loader
           - os: windows-latest
+            go-version: 1.21.x
+            init_script: cmd /c init.bat
             gen_script: cmd /c gen.bat
-            cmake_config: cmake -S . -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Debug
             run_loader: .\bin\loader.exe
 
+    name: test (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
 
     steps:
       - name: Checkout Code
         uses: actions/checkout@v6
+        with:
+          submodules: recursive
 
       - name: Install Go
         uses: actions/setup-go@v6
         with:
-          go-version: 1.21.x
+          go-version: ${{ matrix.go-version }}
           cache: true
 
       - name: Install dependencies (Ubuntu)
@@ -52,16 +57,15 @@ jobs:
         uses: ilammy/msvc-dev-cmd@v1
 
       - name: Init submodules and build protobuf
-        shell: bash
-        run: bash init.sh
+        run: ${{ matrix.init_script }}
 
       - name: Generate protoconf
         working-directory: test/cpp-tableau-loader
         run: ${{ matrix.gen_script }}
 
       - name: CMake Configure
         working-directory: test/cpp-tableau-loader
-        run: ${{ matrix.cmake_config }}
+        run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17
 
       - name: CMake Build
         working-directory: test/cpp-tableau-loader

.github/workflows/testing-csharp.yml

@@ -18,43 +18,38 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
+            go-version: 1.21.x
             gen_script: bash gen.sh
           - os: windows-latest
+            go-version: 1.21.x
             gen_script: cmd /c gen.bat
 
+    name: test (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
 
     steps:
       - name: Checkout Code
         uses: actions/checkout@v6
+        with:
+          submodules: recursive
 
       - name: Install Go
         uses: actions/setup-go@v6
         with:
-          go-version: 1.21.x
+          go-version: ${{ matrix.go-version }}
           cache: true
 
       - name: Install .NET SDK
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: "8.0.x"
 
-      - name: Install dependencies (Ubuntu)
-        if: runner.os == 'Linux'
-        run: sudo apt-get update && sudo apt-get install -y cmake ninja-build
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        run: choco install cmake ninja -y
-
-      - name: Setup MSVC (Windows)
-        if: runner.os == 'Windows'
-        uses: ilammy/msvc-dev-cmd@v1
-
-      - name: Init submodules and build protobuf
-        shell: bash
-        run: bash init.sh
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: "3.19.3"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Generate protoconf
         working-directory: test/csharp-tableau-loader

.github/workflows/testing-go.yml

@@ -16,10 +16,17 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.21.x]
-        os: [ubuntu-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            go-version: 1.21.x
+            gen_script: bash gen.sh
+          - os: windows-latest
+            go-version: 1.21.x
+            gen_script: cmd /c gen.bat
 
+    name: test (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
 
     steps:
       - name: Checkout Code
@@ -34,17 +41,17 @@ jobs:
           cache: true
 
       - name: Install Protoc
-        uses: arduino/setup-protoc@v3
+        uses: arduino/setup-protoc@v1
         with:
-          version: "23.x"
+          version: "3.19.3"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install protoc-gen-go
         run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
 
       - name: Generate protoconf
-        shell: bash
-        run: PROTOC=protoc bash test/go-tableau-loader/gen.sh
+        working-directory: test/go-tableau-loader
+        run: ${{ matrix.gen_script }}
 
       - name: Vet
         run: go vet ./...

cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc

@@ -49,7 +49,7 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p
 bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt,
                        std::shared_ptr<const MessagerOptions> options /* = nullptr*/) {
   options = options ? options : std::make_shared<MessagerOptions>();
-  const std::string& name = msg.GetDescriptor()->name();
+  const std::string name(msg.GetDescriptor()->name());
   std::filesystem::path path;
   if (!options->path.empty()) {
     // path specified in Paths, then use it instead of dir.
@@ -79,7 +79,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem
     // ignore patch files when LoadMode::kModeOnlyMain specified
     return load_func(msg, path, fmt, nullptr);
   }
-  const std::string& name = msg.GetDescriptor()->name();
+  const std::string name(msg.GetDescriptor()->name());
   std::vector<std::filesystem::path> patch_paths;
   if (!options->patch_paths.empty()) {
     // patch path specified in PatchPaths, then use it instead of PatchDirs.
@@ -155,21 +155,21 @@ bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Forma
       parse_options.ignore_unknown_fields = options->GetIgnoreUnknownFields();
       auto status = google::protobuf::util::JsonStringToMessage(content, &msg, parse_options);
       if (!status.ok()) {
-        SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString());
+        SetErrMsg("failed to parse " + std::string(msg.GetDescriptor()->name()) + kJSONExt + ": " + status.ToString());
         return false;
       }
       return true;
     }
     case Format::kText: {
       if (!google::protobuf::TextFormat::ParseFromString(content, &msg)) {
-        SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt);
+        SetErrMsg("failed to parse " + std::string(msg.GetDescriptor()->name()) + kTextExt);
         return false;
       }
       return true;
     }
     case Format::kBin: {
       if (!msg.ParseFromString(content)) {
-        SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt);
+        SetErrMsg("failed to parse " + std::string(msg.GetDescriptor()->name()) + kBinExt);
         return false;
       }
       return true;

cmd/protoc-gen-cpp-tableau-loader/embed/templates/hub.pc.cc.tpl

@@ -54,7 +54,12 @@ void Hub::InitScheduler() {
 std::shared_ptr<MessagerMap> Hub::InternalLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */,
                                                std::shared_ptr<const load::Options> options /* = nullptr */) const {
   // intercept protobuf error logs
+#if TABLEAU_PB_LOG_LEGACY
   auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler);
+#else
+  util::ProtobufAbslLogSink log_sink;
+  absl::AddLogSink(&log_sink);
+#endif
   auto msger_map = NewMessagerMap();
   options = options ? options : std::make_shared<load::Options>();
   for (auto iter : *msger_map) {
@@ -65,14 +70,22 @@ std::shared_ptr<MessagerMap> Hub::InternalLoad(const std::filesystem::path& dir,
     if (!ok) {
       ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str());
       // restore to old protobuf log handler
+#if TABLEAU_PB_LOG_LEGACY
       google::protobuf::SetLogHandler(old_handler);
+#else
+      absl::RemoveLogSink(&log_sink);
+#endif
       return nullptr;
     }
     ATOM_DEBUG("loaded %s", name.c_str());
   }
 
   // restore to old protobuf log handler
+#if TABLEAU_PB_LOG_LEGACY
   google::protobuf::SetLogHandler(old_handler);
+#else
+  absl::RemoveLogSink(&log_sink);
+#endif
   return msger_map;
 }
 

cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc

@@ -83,8 +83,8 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag
   // Ensure both messages are of the same type
   if (dst_descriptor != src_descriptor) {
     SetErrMsg("dst and src are not messages with the same descriptor");
-    ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(),
-               src_descriptor->name().c_str());
+    ATOM_ERROR("dst %s and src %s are not messages with the same descriptor",
+               std::string(dst_descriptor->name()).c_str(), std::string(src_descriptor->name()).c_str());
     return false;
   }
 
@@ -206,18 +206,51 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag
   return true;
 }
 
-// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h
+#if TABLEAU_PB_LOG_LEGACY
+// refer: https://github.com/protocolbuffers/protobuf/blob/v3.19.3/src/google/protobuf/stubs/logging.h
 void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) {
-  static const std::unordered_map<int, log::Level> kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo},
-                                                                {google::protobuf::LOGLEVEL_WARNING, log::kWarn},
-                                                                {google::protobuf::LOGLEVEL_ERROR, log::kError},
-                                                                {google::protobuf::LOGLEVEL_FATAL, log::kFatal}};
+#define TABLEAU_PB_LOG_INFO google::protobuf::LOGLEVEL_INFO
+#define TABLEAU_PB_LOG_WARNING google::protobuf::LOGLEVEL_WARNING
+#define TABLEAU_PB_LOG_ERROR google::protobuf::LOGLEVEL_ERROR
+#define TABLEAU_PB_LOG_FATAL google::protobuf::LOGLEVEL_FATAL
+#define TABLEAU_PB_LOG_LEVEL level
+#define TABLEAU_PB_LOG_FILENAME filename
+#define TABLEAU_PB_LOG_LINE line
+#define TABLEAU_PB_LOG_MESSAGE msg
+#else
+// refer: https://github.com/abseil/abseil-cpp/blob/20250512.1/absl/log/log_entry.h
+void ProtobufAbslLogSink::Send(const absl::LogEntry& entry) {
+#define TABLEAU_PB_LOG_INFO absl::LogSeverity::kInfo
+#define TABLEAU_PB_LOG_WARNING absl::LogSeverity::kWarning
+#define TABLEAU_PB_LOG_ERROR absl::LogSeverity::kError
+#define TABLEAU_PB_LOG_FATAL absl::LogSeverity::kFatal
+#define TABLEAU_PB_LOG_LEVEL entry.log_severity()
+#define TABLEAU_PB_LOG_FILENAME std::string(entry.source_filename()).c_str()
+#define TABLEAU_PB_LOG_LINE entry.source_line()
+#define TABLEAU_PB_LOG_MESSAGE std::string(entry.text_message()).c_str()
+#endif
+
+  static const std::unordered_map<decltype(TABLEAU_PB_LOG_LEVEL), log::Level> kLevelMap = {
+      {TABLEAU_PB_LOG_INFO, log::kInfo},
+      {TABLEAU_PB_LOG_WARNING, log::kWarn},
+      {TABLEAU_PB_LOG_ERROR, log::kError},
+      {TABLEAU_PB_LOG_FATAL, log::kFatal}};
   log::Level lvl = log::kWarn;  // default
-  auto iter = kLevelMap.find(level);
+  auto iter = kLevelMap.find(TABLEAU_PB_LOG_LEVEL);
   if (iter != kLevelMap.end()) {
     lvl = iter->second;
   }
-  ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str());
+  ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", TABLEAU_PB_LOG_FILENAME,
+                   TABLEAU_PB_LOG_LINE, TABLEAU_PB_LOG_MESSAGE);
+
+#undef TABLEAU_PB_LOG_INFO
+#undef TABLEAU_PB_LOG_WARNING
+#undef TABLEAU_PB_LOG_ERROR
+#undef TABLEAU_PB_LOG_FATAL
+#undef TABLEAU_PB_LOG_LEVEL
+#undef TABLEAU_PB_LOG_FILENAME
+#undef TABLEAU_PB_LOG_LINE
+#undef TABLEAU_PB_LOG_MESSAGE
 }
 }  // namespace util
 }  // namespace tableau

cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h

@@ -1,10 +1,21 @@
 #pragma once
 #include <google/protobuf/message.h>
+#include <google/protobuf/stubs/common.h>
 
 #include <chrono>
 #include <filesystem>
 #include <string>
 
+// Protobuf versions before v4.22.0 (GOOGLE_PROTOBUF_VERSION < 4022000) use the legacy
+// logging interface (LogLevel, SetLogHandler). Newer versions removed it in favor of Abseil logging.
+#define TABLEAU_PB_LOG_LEGACY (GOOGLE_PROTOBUF_VERSION < 4022000)
+
+#if !TABLEAU_PB_LOG_LEGACY
+#include <absl/log/log_entry.h>
+#include <absl/log/log_sink.h>
+#include <absl/log/log_sink_registry.h>
+#endif
+
 namespace tableau {
 const std::string& GetErrMsg();
 void SetErrMsg(const std::string& msg);
@@ -57,7 +68,18 @@ const std::string& Format2Ext(Format fmt);
 // PatchMessage patches src into dst, which must be a message with the same descriptor.
 bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src);
 
+#if TABLEAU_PB_LOG_LEGACY
+// ProtobufLogHandler redirects protobuf internal logs to tableau logger.
+// Only available for protobuf < v4.22.0, as newer versions removed the old logging interface.
 void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg);
+#else
+// ProtobufAbslLogSink redirects protobuf/Abseil logs to tableau logger.
+// For protobuf >= v4.22.0, which uses Abseil logging (absl::LogSink).
+class ProtobufAbslLogSink : public absl::LogSink {
+ public:
+  void Send(const absl::LogEntry& entry) override;
+};
+#endif
 
 class TimeProfiler {
  protected:

cmd/protoc-gen-cpp-tableau-loader/helper/helper.go

@@ -26,6 +26,7 @@ func GenerateCommonHeader(gen *protogen.Plugin, g *protogen.GeneratedFile, versi
 	g.P("// versions:")
 	g.P("// - protoc-gen-cpp-tableau-loader v", version)
 	g.P("// - protoc                        ", protocVersion(gen))
+	g.P("// clang-format off")
 }
 
 func protocVersion(gen *protogen.Plugin) string {

cmd/protoc-gen-cpp-tableau-loader/messager.go

@@ -160,7 +160,7 @@ func genCppMessage(g *protogen.GeneratedFile, message *protogen.Message) {
 	orderedMapGenerator := orderedmap.NewGenerator(g, message)
 	indexGenerator := indexes.NewGenerator(g, indexDescriptor, message)
 
-	g.P("const std::string ", messagerName, "::kProtoName = ", cppFullName, `::GetDescriptor()->name();`)
+	g.P("const std::string ", messagerName, "::kProtoName = std::string(", cppFullName, `::GetDescriptor()->name());`)
 	g.P()
 	g.P("bool ", messagerName, "::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr<const load::MessagerOptions> options /* = nullptr */) {")
 	g.P(helper.Indent(1), "tableau::util::TimeProfiler profiler;")

init.bat

@@ -13,21 +13,22 @@ cd /d "%repoRoot%"
 
 git submodule update --init --recursive
 
-REM google protobuf
-cd third_party\_submodules\protobuf
-git checkout v3.19.3
-git submodule update --init --recursive
-
 REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc)
 REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration
-cd cmake
+cd third_party\_submodules\protobuf\cmake
 REM use Debug version
 REM - protobuf_MSVC_STATIC_RUNTIME defaults to ON, which uses static CRT (/MTd for Debug).
 REM   Our project's CMakeLists.txt also sets static CRT to match.
 REM - protobuf_WITH_ZLIB=OFF: disable ZLIB dependency to avoid ZLIB::ZLIB link requirement
 REM   in protobuf's exported CMake targets, which simplifies cross-platform builds.
 REM - protobuf_BUILD_SHARED_LIBS=OFF: build static libraries explicitly.
-cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF
+cmake -S . -B build -G Ninja ^
+  -DCMAKE_BUILD_TYPE=Debug ^
+  -DCMAKE_CXX_STANDARD=17 ^
+  -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ^
+  -Dprotobuf_BUILD_TESTS=OFF ^
+  -Dprotobuf_WITH_ZLIB=OFF ^
+  -Dprotobuf_BUILD_SHARED_LIBS=OFF
 
 REM Compile the code
 cmake --build build --parallel