From 6d92a3ddaa2b92fa153a9558e04c8c93327f19ff Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 30 Jun 2025 09:34:54 +0100 Subject: [PATCH 01/16] Add cluster support (#228) ## Release notes: usage and product changes Introduce cluster support for TypeDB 3.x, featuring server replicas and the new version message. ## Implementation Remove database replicas. Instead, add server replicas (by extending the usual `Server` message) with a similar set of fields. Instead of adding a `is_primary` flag, introduce an extensible enum `ReplicaType` for more granular split between primary and supporting nodes. To reduce the network overhead, this new information is provided in the initial server's connection response. --- proto/BUILD | 1 + proto/connection.proto | 9 ++------- proto/database.proto | 20 ++++---------------- proto/error.proto | 2 +- proto/server.proto | 20 +++++++++++++++++++- proto/typedb-service.proto | 3 +++ 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/proto/BUILD b/proto/BUILD index 1eaa241..d6e31ac 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -66,6 +66,7 @@ proto_library( deps = [ ":authentication-proto", ":database-proto", + ":server-proto", ":version-proto" ], ) diff --git a/proto/connection.proto b/proto/connection.proto index d288eb4..b083d7f 100644 --- a/proto/connection.proto +++ b/proto/connection.proto @@ -6,35 +6,30 @@ syntax = "proto3"; import "proto/authentication.proto"; import "proto/database.proto"; +import "proto/server.proto"; import "proto/version.proto"; package typedb.protocol; message Connection { - message Open { message Req { Version version = 1; ExtensionVersion extension_version = 5; string driver_lang = 2; string driver_version = 3; - Authentication.Token.Create.Req authentication = 4; } message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - - // pre-send all databases and replica info - DatabaseManager.All.Res databases_all = 3; - + ServerManager.All.Res servers_all = 3; Authentication.Token.Create.Res authentication = 4; } } } -// Connection ID and Token are expected in all message metadata message ConnectionID { bytes id = 1; } diff --git a/proto/database.proto b/proto/database.proto index eefa6d4..4694092 100644 --- a/proto/database.proto +++ b/proto/database.proto @@ -16,14 +16,14 @@ message DatabaseManager { } message Res { - DatabaseReplicas database = 1; + Database database = 1; } } message All { message Req {} message Res { - repeated DatabaseReplicas databases = 1; + repeated Database databases = 1; } } @@ -43,7 +43,7 @@ message DatabaseManager { } message Res { - DatabaseReplicas database = 1; + Database database = 1; } } @@ -58,20 +58,8 @@ message DatabaseManager { } } -message DatabaseReplicas { - - string name = 1; - repeated Replica replicas = 2; - - message Replica { - string address = 1; - bool primary = 2; - bool preferred = 3; - int64 term = 4; - } -} - message Database { + string name = 1; message Schema { message Req { diff --git a/proto/error.proto b/proto/error.proto index eaa7a55..7543d28 100644 --- a/proto/error.proto +++ b/proto/error.proto @@ -7,7 +7,7 @@ syntax = "proto3"; package typedb.protocol; // This is an emulation of the google ErrorDetails message. Generally, ErrorDetails are submitted via the GRPC error -// mechanism, but a manual error sending is required in streams +// mechanism, but a manual error sending is useful to differentiate error levels in streams message Error { string error_code = 1; string domain = 2; diff --git a/proto/server.proto b/proto/server.proto index 40d2c98..b1c0cc3 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -7,7 +7,6 @@ syntax = "proto3"; package typedb.protocol; message ServerManager { - message All { message Req {} message Res { @@ -18,4 +17,23 @@ message ServerManager { message Server { string address = 1; + optional ReplicaStatus replica_status = 2; + + message ReplicaStatus { + ReplicaType replica_type = 1; + int64 term = 2; + + enum ReplicaType { + Primary = 0; + Secondary = 1; + } + } + + message Version { + message Req {} + message Res { + string distribution = 1; + string version = 2; + } + } } diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto index 95f0c23..79e6936 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb-service.proto @@ -25,6 +25,9 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + // Server API + rpc server_version (Server.Version.Req) returns (Server.Version.Res); + // User Manager API rpc users_get (UserManager.Get.Req) returns (UserManager.Get.Res); rpc users_all (UserManager.All.Req) returns (UserManager.All.Res); From 6e473ebf84e5482701529f919a2d0018c796f3f0 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Thu, 3 Jul 2025 16:20:59 +0100 Subject: [PATCH 02/16] Add server register/deregister endpoint --- proto/server.proto | 16 ++++++++++++++++ proto/typedb-service.proto | 2 ++ 2 files changed, 18 insertions(+) diff --git a/proto/server.proto b/proto/server.proto index b1c0cc3..7dd1914 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -13,6 +13,21 @@ message ServerManager { repeated Server servers = 1; } } + + message Register { + message Req { + uint64 replica_id = 1; + string address = 2; + } + message Res {} + } + + message Deregister { + message Req { + uint64 replica_id = 1; + } + message Res {} + } } message Server { @@ -20,6 +35,7 @@ message Server { optional ReplicaStatus replica_status = 2; message ReplicaStatus { + uint64 replica_id = 3; ReplicaType replica_type = 1; int64 term = 2; diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto index 79e6936..3b401c7 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb-service.proto @@ -24,6 +24,8 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + rpc servers_register(ServerManager.Register.Req) returns (ServerManager.Register.Res); + rpc servers_deregister(ServerManager.Deregister.Req) returns (ServerManager.Deregister.Res); // Server API rpc server_version (Server.Version.Req) returns (Server.Version.Res); From a3f6586d5e91c52e617db1304941a90aa6d57676 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Thu, 3 Jul 2025 16:24:49 +0100 Subject: [PATCH 03/16] Update the numbering of ReplicaStatus fields --- proto/server.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proto/server.proto b/proto/server.proto index 7dd1914..980731f 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -35,9 +35,9 @@ message Server { optional ReplicaStatus replica_status = 2; message ReplicaStatus { - uint64 replica_id = 3; - ReplicaType replica_type = 1; - int64 term = 2; + uint64 replica_id = 1; + ReplicaType replica_type = 2; + int64 term = 3; enum ReplicaType { Primary = 0; From 3088a14247ef31cae8107f85e335f96991fedd95 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Fri, 4 Jul 2025 09:29:27 +0100 Subject: [PATCH 04/16] Change type of term from in64 to uint64 --- proto/server.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/server.proto b/proto/server.proto index 980731f..07afb72 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -37,7 +37,7 @@ message Server { message ReplicaStatus { uint64 replica_id = 1; ReplicaType replica_type = 2; - int64 term = 3; + uint64 term = 3; enum ReplicaType { Primary = 0; From aad93a5abff6842838844bda231609aeb2cfc7a0 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Wed, 9 Jul 2025 13:51:43 +0100 Subject: [PATCH 05/16] Move raft peering protocol to this repo --- grpc/rust/BUILD | 44 +++++++++++++++++--------------- grpc/rust/build.rs | 2 ++ proto/BUILD | 29 +++++++++++++++------ proto/raft-peering-service.proto | 9 +++++++ proto/raft-peering.proto | 12 +++++++++ 5 files changed, 67 insertions(+), 29 deletions(-) create mode 100644 proto/raft-peering-service.proto create mode 100644 proto/raft-peering.proto diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index d3e84fe..6d9fbdd 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -10,64 +10,66 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co rust_tonic_compile( name = "typedb_protocol_src", - packages = ["typedb.protocol"], srcs = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", + "//proto:analyze-proto", + "//proto:analyzed-conjunction-proto", "//proto:answer-proto", + "//proto:authentication-proto", "//proto:concept-proto", "//proto:connection-proto", - "//proto:authentication-proto", + "//proto:database-proto", + "//proto:error-proto", + "//proto:migration-proto", "//proto:options-proto", "//proto:query-proto", + "//proto:raft-peering-proto", + "//proto:raft-peering-service", + "//proto:server-proto", "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", + "//proto:typedb-service", + "//proto:user-proto", "//proto:version-proto", - ] + ], + packages = ["typedb.protocol"], ) rust_library( name = "typedb_protocol", - crate_root = ":typedb_protocol_src", srcs = [ ":typedb_protocol_src", ], - deps = [ - "@crates//:tonic", - "@crates//:prost", - ], + crate_root = ":typedb_protocol_src", tags = [ "crate-name=typedb-protocol", ], visibility = ["//visibility:public"], + deps = [ + "@crates//:prost", + "@crates//:tonic", + ], ) assemble_crate( name = "assemble_crate", - target = ":typedb_protocol", description = "TypeDB Protocol", homepage = "https://github.com/typedb/typedb-protocol", - readme_file = "//:README.md", license = "MPL-2.0", license_file = "//:LICENSE", + readme_file = "//:README.md", repository = "https://github.com/typedb/typedb-protocol", + target = ":typedb_protocol", ) deploy_crate( name = "deploy_crate", - target = ":assemble_crate", + release = deployment["crate"]["release"], snapshot = deployment["crate"]["snapshot"], - release = deployment["crate"]["release"] + target = ":assemble_crate", ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs index 8ea60f8..ad54f3c 100644 --- a/grpc/rust/build.rs +++ b/grpc/rust/build.rs @@ -20,6 +20,8 @@ fn main() -> std::io::Result<()> { "../../proto/typedb-service.proto", "../../proto/user.proto", "../../proto/version.proto", + "../../proto/raft-peering.proto", + "../../proto/raft-peering-service.proto", ]; tonic_build::configure() diff --git a/proto/BUILD b/proto/BUILD index d6e31ac..fbdd41b 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -6,20 +6,33 @@ package(default_visibility = ["//visibility:public"]) load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") +proto_library( + name = "raft-peering-service", + srcs = [":raft-peering-service.proto"], + deps = [ + ":raft-peering-proto", + ], +) + proto_library( name = "typedb-service", srcs = [":typedb-service.proto"], deps = [ - ":connection-proto", ":authentication-proto", - ":server-proto", - ":user-proto", + ":connection-proto", ":database-proto", - ":transaction-proto", ":migration-proto", + ":server-proto", + ":transaction-proto", + ":user-proto", ], ) +proto_library( + name = "raft-peering-proto", + srcs = [":raft-peering.proto"], +) + proto_library( name = "server-proto", srcs = [":server.proto"], @@ -67,7 +80,7 @@ proto_library( ":authentication-proto", ":database-proto", ":server-proto", - ":version-proto" + ":version-proto", ], ) @@ -103,7 +116,7 @@ proto_library( ":error-proto", ":options-proto", ":query-proto", - ] + ], ) proto_library( @@ -146,12 +159,12 @@ filegroup( "query.proto", "transaction.proto", "version.proto", - ] + ], ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/proto/raft-peering-service.proto b/proto/raft-peering-service.proto new file mode 100644 index 0000000..e87a8e0 --- /dev/null +++ b/proto/raft-peering-service.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +import "proto/raft-peering.proto"; + +package typedb.protocol; + +service RaftPeering { + rpc Peering(PeeringRequest) returns (PeeringResponse); +} \ No newline at end of file diff --git a/proto/raft-peering.proto b/proto/raft-peering.proto new file mode 100644 index 0000000..33a1add --- /dev/null +++ b/proto/raft-peering.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package typedb.protocol; + +message PeeringRequest { + uint64 from_id = 1; + string from_address = 2; + bytes message = 3; +} + +message PeeringResponse { +} From a78e444b07d839b5148fe848f72afc7e08fd112f Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 21 Jul 2025 16:46:09 +0100 Subject: [PATCH 06/16] Add server_status message to the raft peering service (#229) ## Release notes: usage and product changes Add server_status message to the raft peering service and rename the old raft peering messages to adjust to the existing naming conventions. ## Implementation --- proto/BUILD | 3 +++ proto/raft-peering-service.proto | 7 ++++++- proto/raft-peering.proto | 27 ++++++++++++++++++++++----- proto/server.proto | 3 ++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/proto/BUILD b/proto/BUILD index fbdd41b..4be34bf 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -31,6 +31,9 @@ proto_library( proto_library( name = "raft-peering-proto", srcs = [":raft-peering.proto"], + deps = [ + ":server-proto", + ], ) proto_library( diff --git a/proto/raft-peering-service.proto b/proto/raft-peering-service.proto index e87a8e0..82301fd 100644 --- a/proto/raft-peering-service.proto +++ b/proto/raft-peering-service.proto @@ -1,3 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + syntax = "proto3"; import "proto/raft-peering.proto"; @@ -5,5 +9,6 @@ import "proto/raft-peering.proto"; package typedb.protocol; service RaftPeering { - rpc Peering(PeeringRequest) returns (PeeringResponse); + rpc peering (Peering.Req) returns (Peering.Res); + rpc server_status (ServerStatus.Req) returns (ServerStatus.Res); } \ No newline at end of file diff --git a/proto/raft-peering.proto b/proto/raft-peering.proto index 33a1add..cc6411e 100644 --- a/proto/raft-peering.proto +++ b/proto/raft-peering.proto @@ -1,12 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + syntax = "proto3"; package typedb.protocol; -message PeeringRequest { - uint64 from_id = 1; - string from_address = 2; - bytes message = 3; +import "proto/server.proto"; + +// TODO: Rename to Replication +message Peering { + message Req { + uint64 from_id = 1; + string from_address = 2; + bytes message = 3; + } + + message Res {} } -message PeeringResponse { +message ServerStatus { + message Req {} + + message Res { + string connection_address = 1; + Server.ReplicaStatus replica_status = 2; + } } diff --git a/proto/server.proto b/proto/server.proto index 07afb72..e29d01e 100644 --- a/proto/server.proto +++ b/proto/server.proto @@ -41,7 +41,8 @@ message Server { enum ReplicaType { Primary = 0; - Secondary = 1; + Candidate = 1; + Secondary = 2; } } From af19442e7c2b8367c1c568b4bb14a2075c00b7b4 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Thu, 14 Aug 2025 14:47:42 +0100 Subject: [PATCH 07/16] Split protocol into 3 separate services (#230) Split the protocol into 3 services: * TypeDBService (contains common TypeDB logic): server, database, user, and transaction management, etc. * TypeDBClusteringService (contains TypeDB logic regarding clustering): replica registration, server status. * RaftService (contains Raft logic): replication operations. Split the old RaftPeeringService into TypeDBClusteringService and RaftService. Move separate services and their implementation files into separate subpackages. Rename methods. Introduce `ServersGet` method for single server status retrieval. Introduce `registration` method to verify the replica registration operation before submitting it to Raft. --- grpc/java/BUILD | 34 ++-- grpc/nodejs/BUILD | 34 ++-- grpc/rust/BUILD | 36 +++-- grpc/rust/build.rs | 36 +++-- proto/BUILD | 152 +----------------- proto/raft_service/BUILD | 28 ++++ proto/raft_service/raft_service.proto | 14 ++ .../replication.proto} | 14 +- proto/typedb_clustering_service/BUILD | 30 ++++ .../typedb_clustering.proto | 25 +++ .../typedb_clustering_service.proto} | 8 +- proto/typedb_service/BUILD | 134 +++++++++++++++ proto/{ => typedb_service}/analyze.proto | 8 +- .../analyzed-conjunction.proto | 4 +- proto/{ => typedb_service}/answer.proto | 2 +- .../{ => typedb_service}/authentication.proto | 0 proto/{ => typedb_service}/concept.proto | 0 proto/{ => typedb_service}/connection.proto | 6 +- proto/{ => typedb_service}/database.proto | 2 +- proto/{ => typedb_service}/error.proto | 0 proto/{ => typedb_service}/migration.proto | 2 +- proto/{ => typedb_service}/options.proto | 0 proto/{ => typedb_service}/query.proto | 8 +- proto/{ => typedb_service}/server.proto | 7 + proto/{ => typedb_service}/transaction.proto | 8 +- .../typedb_service.proto} | 15 +- proto/{ => typedb_service}/user.proto | 0 27 files changed, 355 insertions(+), 252 deletions(-) create mode 100644 proto/raft_service/BUILD create mode 100644 proto/raft_service/raft_service.proto rename proto/{raft-peering.proto => raft_service/replication.proto} (61%) create mode 100644 proto/typedb_clustering_service/BUILD create mode 100644 proto/typedb_clustering_service/typedb_clustering.proto rename proto/{raft-peering-service.proto => typedb_clustering_service/typedb_clustering_service.proto} (66%) create mode 100644 proto/typedb_service/BUILD rename proto/{ => typedb_service}/analyze.proto (95%) rename proto/{ => typedb_service}/analyzed-conjunction.proto (97%) rename proto/{ => typedb_service}/answer.proto (97%) rename proto/{ => typedb_service}/authentication.proto (100%) rename proto/{ => typedb_service}/concept.proto (100%) rename proto/{ => typedb_service}/connection.proto (84%) rename proto/{ => typedb_service}/database.proto (97%) rename proto/{ => typedb_service}/error.proto (100%) rename proto/{ => typedb_service}/migration.proto (98%) rename proto/{ => typedb_service}/options.proto (100%) rename proto/{ => typedb_service}/query.proto (92%) rename proto/{ => typedb_service}/server.proto (91%) rename proto/{ => typedb_service}/transaction.proto (93%) rename proto/{typedb-service.proto => typedb_service/typedb_service.proto} (86%) rename proto/{ => typedb_service}/user.proto (100%) diff --git a/grpc/java/BUILD b/grpc/java/BUILD index cf6f231..a793641 100644 --- a/grpc/java/BUILD +++ b/grpc/java/BUILD @@ -9,22 +9,26 @@ load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") java_grpc_library( name = "typedb-protocol", protos = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", - "//proto:answer-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:authentication-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ], # TypeDB Core bundles JARs by maven coordinate, we can remove this when Core is rewritten in Rust tags = ["maven_coordinates=com.typedb:typedb-protocol:{pom_version}"], diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD index c9b8827..8e00bae 100644 --- a/grpc/nodejs/BUILD +++ b/grpc/nodejs/BUILD @@ -25,22 +25,26 @@ protoc_gen_ts_bin.protoc_gen_ts_binary( ts_grpc_compile( name = "typedb-protocol-src", deps = [ - "//proto:typedb-service", - "//proto:server-proto", - "//proto:user-proto", - "//proto:database-proto", - "//proto:migration-proto", - "//proto:error-proto", - "//proto:answer-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:authentication-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:transaction-proto", - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ] ) diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index 6d9fbdd..200ec89 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -11,24 +11,26 @@ load("@typedb_dependencies//builder/proto_grpc/rust:compile.bzl", "rust_tonic_co rust_tonic_compile( name = "typedb_protocol_src", srcs = [ - "//proto:analyze-proto", - "//proto:analyzed-conjunction-proto", - "//proto:answer-proto", - "//proto:authentication-proto", - "//proto:concept-proto", - "//proto:connection-proto", - "//proto:database-proto", - "//proto:error-proto", - "//proto:migration-proto", - "//proto:options-proto", - "//proto:query-proto", - "//proto:raft-peering-proto", - "//proto:raft-peering-service", - "//proto:server-proto", - "//proto:transaction-proto", - "//proto:typedb-service", - "//proto:user-proto", "//proto:version-proto", + "//proto/raft_service:raft-service", + "//proto/raft_service:replication-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_service:analyze-proto", + "//proto/typedb_service:analyzed-conjunction-proto", + "//proto/typedb_service:answer-proto", + "//proto/typedb_service:authentication-proto", + "//proto/typedb_service:concept-proto", + "//proto/typedb_service:connection-proto", + "//proto/typedb_service:database-proto", + "//proto/typedb_service:error-proto", + "//proto/typedb_service:migration-proto", + "//proto/typedb_service:options-proto", + "//proto/typedb_service:query-proto", + "//proto/typedb_service:server-proto", + "//proto/typedb_service:transaction-proto", + "//proto/typedb_service:typedb-service", + "//proto/typedb_service:user-proto", ], packages = ["typedb.protocol"], ) diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs index ad54f3c..8ba15f0 100644 --- a/grpc/rust/build.rs +++ b/grpc/rust/build.rs @@ -4,24 +4,26 @@ fn main() -> std::io::Result<()> { let protos = vec![ - "../../proto/analyze.proto", - "../../proto/answer.proto", - "../../proto/authentication.proto", - "../../proto/concept.proto", - "../../proto/connection.proto", - "../../proto/analyzed-conjunction.proto", - "../../proto/database.proto", - "../../proto/error.proto", - "../../proto/migration.proto", - "../../proto/options.proto", - "../../proto/query.proto", - "../../proto/server.proto", - "../../proto/transaction.proto", - "../../proto/typedb-service.proto", - "../../proto/user.proto", + "../../proto/raft_service/replication.proto", + "../../proto/raft_service/raft_service.proto", + "../../proto/typedb_clustering_service/typedb_clustering.proto", + "../../proto/typedb_clustering_service/typedb_clustering_service.proto", + "../../proto/typedb_service/analyze.proto", + "../../proto/typedb_service/analyzed-conjunction.proto", + "../../proto/typedb_service/answer.proto", + "../../proto/typedb_service/authentication.proto", + "../../proto/typedb_service/concept.proto", + "../../proto/typedb_service/connection.proto", + "../../proto/typedb_service/database.proto", + "../../proto/typedb_service/error.proto", + "../../proto/typedb_service/migration.proto", + "../../proto/typedb_service/options.proto", + "../../proto/typedb_service/query.proto", + "../../proto/typedb_service/server.proto", + "../../proto/typedb_service/transaction.proto", + "../../proto/typedb_service/typedb_service.proto", + "../../proto/typedb_service/user.proto", "../../proto/version.proto", - "../../proto/raft-peering.proto", - "../../proto/raft-peering-service.proto", ]; tonic_build::configure() diff --git a/proto/BUILD b/proto/BUILD index 4be34bf..3b6d63b 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -6,142 +6,6 @@ package(default_visibility = ["//visibility:public"]) load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") -proto_library( - name = "raft-peering-service", - srcs = [":raft-peering-service.proto"], - deps = [ - ":raft-peering-proto", - ], -) - -proto_library( - name = "typedb-service", - srcs = [":typedb-service.proto"], - deps = [ - ":authentication-proto", - ":connection-proto", - ":database-proto", - ":migration-proto", - ":server-proto", - ":transaction-proto", - ":user-proto", - ], -) - -proto_library( - name = "raft-peering-proto", - srcs = [":raft-peering.proto"], - deps = [ - ":server-proto", - ], -) - -proto_library( - name = "server-proto", - srcs = [":server.proto"], -) - -proto_library( - name = "database-proto", - srcs = [":database.proto"], - deps = [ - ":migration-proto", - ], -) - -proto_library( - name = "migration-proto", - srcs = [":migration.proto"], - deps = [":concept-proto"], -) - -proto_library( - name = "user-proto", - srcs = [":user.proto"], -) - -proto_library( - name = "answer-proto", - srcs = ["answer.proto"], - deps = [":concept-proto"], -) - -proto_library( - name = "authentication-proto", - srcs = ["authentication.proto"], -) - -proto_library( - name = "concept-proto", - srcs = ["concept.proto"], -) - -proto_library( - name = "connection-proto", - srcs = ["connection.proto"], - deps = [ - ":authentication-proto", - ":database-proto", - ":server-proto", - ":version-proto", - ], -) - -proto_library( - name = "options-proto", - srcs = ["options.proto"], -) - -proto_library( - name = "query-proto", - srcs = ["query.proto"], - deps = [ - ":analyze-proto", - ":answer-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ], -) - -proto_library( - name = "error-proto", - srcs = ["error.proto"], -) - -proto_library( - name = "transaction-proto", - srcs = ["transaction.proto"], - deps = [ - ":analyze-proto", - ":answer-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ":query-proto", - ], -) - -proto_library( - name = "analyze-proto", - srcs = ["analyze.proto"], - deps = [ - ":analyzed-conjunction-proto", - ":concept-proto", - ":error-proto", - ":options-proto", - ], -) - -proto_library( - name = "analyzed-conjunction-proto", - srcs = ["analyzed-conjunction.proto"], - deps = [ - ":answer-proto", - ":concept-proto", - ], -) - proto_library( name = "version-proto", srcs = ["version.proto"], @@ -153,14 +17,14 @@ proto_library( filegroup( name = "proto-raw-buffers", srcs = [ - "analyze.proto", - "answer.proto", - "concept.proto", - "connection.proto", - "analyzed-conjunction.proto", - "options.proto", - "query.proto", - "transaction.proto", + "//proto/typedb_service:analyze.proto", + "//proto/typedb_service:analyzed-conjunction.proto", + "//proto/typedb_service:answer.proto", + "//proto/typedb_service:concept.proto", + "//proto/typedb_service:connection.proto", + "//proto/typedb_service:options.proto", + "//proto/typedb_service:query.proto", + "//proto/typedb_service:transaction.proto", "version.proto", ], ) diff --git a/proto/raft_service/BUILD b/proto/raft_service/BUILD new file mode 100644 index 0000000..62000b0 --- /dev/null +++ b/proto/raft_service/BUILD @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "raft-service", + srcs = ["raft_service.proto"], + deps = [ + ":replication-proto", + ], +) + +proto_library( + name = "replication-proto", + srcs = ["replication.proto"], + deps = [], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/raft_service/raft_service.proto b/proto/raft_service/raft_service.proto new file mode 100644 index 0000000..51886f2 --- /dev/null +++ b/proto/raft_service/raft_service.proto @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +import "proto/raft_service/replication.proto"; + +package typedb.protocol; + +// TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`? +service Raft { + rpc replication (Replication.Req) returns (Replication.Res); +} diff --git a/proto/raft-peering.proto b/proto/raft_service/replication.proto similarity index 61% rename from proto/raft-peering.proto rename to proto/raft_service/replication.proto index cc6411e..7d9dc92 100644 --- a/proto/raft-peering.proto +++ b/proto/raft_service/replication.proto @@ -6,10 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/server.proto"; - -// TODO: Rename to Replication -message Peering { +message Replication { message Req { uint64 from_id = 1; string from_address = 2; @@ -18,12 +15,3 @@ message Peering { message Res {} } - -message ServerStatus { - message Req {} - - message Res { - string connection_address = 1; - Server.ReplicaStatus replica_status = 2; - } -} diff --git a/proto/typedb_clustering_service/BUILD b/proto/typedb_clustering_service/BUILD new file mode 100644 index 0000000..4a5caf0 --- /dev/null +++ b/proto/typedb_clustering_service/BUILD @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "typedb-clustering-service", + srcs = ["typedb_clustering_service.proto"], + deps = [ + ":typedb-clustering-proto", + ], +) + +proto_library( + name = "typedb-clustering-proto", + srcs = ["typedb_clustering.proto"], + deps = [ + "//proto/typedb_service:server-proto", + ], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/typedb_clustering_service/typedb_clustering.proto b/proto/typedb_clustering_service/typedb_clustering.proto new file mode 100644 index 0000000..74ab151 --- /dev/null +++ b/proto/typedb_clustering_service/typedb_clustering.proto @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +package typedb.protocol; + +import "proto/typedb_service/server.proto"; + +message CanRegister { + message Req { + uint64 replica_id = 1; + } + + message Res {} +} + +message ServerStatus { + message Req {} + + message Res { + Server server = 1; + } +} diff --git a/proto/raft-peering-service.proto b/proto/typedb_clustering_service/typedb_clustering_service.proto similarity index 66% rename from proto/raft-peering-service.proto rename to proto/typedb_clustering_service/typedb_clustering_service.proto index 82301fd..cc83c61 100644 --- a/proto/raft-peering-service.proto +++ b/proto/typedb_clustering_service/typedb_clustering_service.proto @@ -4,11 +4,11 @@ syntax = "proto3"; -import "proto/raft-peering.proto"; +import "proto/typedb_clustering_service/typedb_clustering.proto"; package typedb.protocol; -service RaftPeering { - rpc peering (Peering.Req) returns (Peering.Res); +service TypeDBClustering { + rpc can_register (CanRegister.Req) returns (CanRegister.Res); rpc server_status (ServerStatus.Req) returns (ServerStatus.Res); -} \ No newline at end of file +} diff --git a/proto/typedb_service/BUILD b/proto/typedb_service/BUILD new file mode 100644 index 0000000..7246b15 --- /dev/null +++ b/proto/typedb_service/BUILD @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package(default_visibility = ["//visibility:public"]) + +load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") + +proto_library( + name = "typedb-service", + srcs = ["typedb_service.proto"], + deps = [ + ":authentication-proto", + ":connection-proto", + ":database-proto", + ":migration-proto", + ":server-proto", + ":transaction-proto", + ":user-proto", + ], +) + +proto_library( + name = "analyze-proto", + srcs = ["analyze.proto"], + deps = [ + ":analyzed-conjunction-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ], +) + +proto_library( + name = "analyzed-conjunction-proto", + srcs = ["analyzed-conjunction.proto"], + deps = [ + ":answer-proto", + ":concept-proto", + ], +) + +proto_library( + name = "answer-proto", + srcs = ["answer.proto"], + deps = [":concept-proto"], +) + +proto_library( + name = "authentication-proto", + srcs = ["authentication.proto"], +) + +proto_library( + name = "concept-proto", + srcs = ["concept.proto"], +) + +proto_library( + name = "connection-proto", + srcs = ["connection.proto"], + deps = [ + ":authentication-proto", + ":database-proto", + ":server-proto", + "//proto:version-proto", + ], +) + +proto_library( + name = "database-proto", + srcs = ["database.proto"], + deps = [ + ":migration-proto", + ], +) + +proto_library( + name = "error-proto", + srcs = ["error.proto"], +) + +proto_library( + name = "migration-proto", + srcs = ["migration.proto"], + deps = [":concept-proto"], +) + +proto_library( + name = "options-proto", + srcs = ["options.proto"], +) + +proto_library( + name = "query-proto", + srcs = ["query.proto"], + deps = [ + ":analyze-proto", + ":answer-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ], +) + +proto_library( + name = "server-proto", + srcs = ["server.proto"], +) + +proto_library( + name = "transaction-proto", + srcs = ["transaction.proto"], + deps = [ + ":analyze-proto", + ":answer-proto", + ":concept-proto", + ":error-proto", + ":options-proto", + ":query-proto", + ], +) + +proto_library( + name = "user-proto", + srcs = ["user.proto"], +) + +checkstyle_test( + name = "checkstyle", + size = "small", + include = glob(["*"]), + license_type = "mpl-header", +) diff --git a/proto/analyze.proto b/proto/typedb_service/analyze.proto similarity index 95% rename from proto/analyze.proto rename to proto/typedb_service/analyze.proto index 7e3eb92..1d3dee8 100644 --- a/proto/analyze.proto +++ b/proto/typedb_service/analyze.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyzed-conjunction.proto"; -import "proto/concept.proto"; -import "proto/error.proto"; -import "proto/options.proto"; +import "proto/typedb_service/analyzed-conjunction.proto"; +import "proto/typedb_service/concept.proto"; +import "proto/typedb_service/error.proto"; +import "proto/typedb_service/options.proto"; package typedb.protocol; diff --git a/proto/analyzed-conjunction.proto b/proto/typedb_service/analyzed-conjunction.proto similarity index 97% rename from proto/analyzed-conjunction.proto rename to proto/typedb_service/analyzed-conjunction.proto index eef88d0..a28f01e 100644 --- a/proto/analyzed-conjunction.proto +++ b/proto/typedb_service/analyzed-conjunction.proto @@ -4,8 +4,8 @@ syntax = "proto3"; -import "proto/answer.proto"; -import "proto/concept.proto"; +import "proto/typedb_service/answer.proto"; +import "proto/typedb_service/concept.proto"; package typedb.protocol; diff --git a/proto/answer.proto b/proto/typedb_service/answer.proto similarity index 97% rename from proto/answer.proto rename to proto/typedb_service/answer.proto index 91d4817..2e88e54 100644 --- a/proto/answer.proto +++ b/proto/typedb_service/answer.proto @@ -4,7 +4,7 @@ syntax = "proto3"; -import "proto/concept.proto"; +import "proto/typedb_service/concept.proto"; package typedb.protocol; diff --git a/proto/authentication.proto b/proto/typedb_service/authentication.proto similarity index 100% rename from proto/authentication.proto rename to proto/typedb_service/authentication.proto diff --git a/proto/concept.proto b/proto/typedb_service/concept.proto similarity index 100% rename from proto/concept.proto rename to proto/typedb_service/concept.proto diff --git a/proto/connection.proto b/proto/typedb_service/connection.proto similarity index 84% rename from proto/connection.proto rename to proto/typedb_service/connection.proto index b083d7f..d4d3207 100644 --- a/proto/connection.proto +++ b/proto/typedb_service/connection.proto @@ -4,9 +4,9 @@ syntax = "proto3"; -import "proto/authentication.proto"; -import "proto/database.proto"; -import "proto/server.proto"; +import "proto/typedb_service/authentication.proto"; +import "proto/typedb_service/database.proto"; +import "proto/typedb_service/server.proto"; import "proto/version.proto"; package typedb.protocol; diff --git a/proto/database.proto b/proto/typedb_service/database.proto similarity index 97% rename from proto/database.proto rename to proto/typedb_service/database.proto index 4694092..e3a2ca0 100644 --- a/proto/database.proto +++ b/proto/typedb_service/database.proto @@ -6,7 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/migration.proto"; +import "proto/typedb_service/migration.proto"; message DatabaseManager { diff --git a/proto/error.proto b/proto/typedb_service/error.proto similarity index 100% rename from proto/error.proto rename to proto/typedb_service/error.proto diff --git a/proto/migration.proto b/proto/typedb_service/migration.proto similarity index 98% rename from proto/migration.proto rename to proto/typedb_service/migration.proto index 46ec24c..ba82617 100644 --- a/proto/migration.proto +++ b/proto/typedb_service/migration.proto @@ -6,7 +6,7 @@ syntax = "proto3"; package typedb.protocol; -import "proto/concept.proto"; +import "proto/typedb_service/concept.proto"; message Migration { message Export { diff --git a/proto/options.proto b/proto/typedb_service/options.proto similarity index 100% rename from proto/options.proto rename to proto/typedb_service/options.proto diff --git a/proto/query.proto b/proto/typedb_service/query.proto similarity index 92% rename from proto/query.proto rename to proto/typedb_service/query.proto index 8fd640b..c8e1347 100644 --- a/proto/query.proto +++ b/proto/typedb_service/query.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyze.proto"; -import "proto/answer.proto"; -import "proto/options.proto"; -import "proto/error.proto"; +import "proto/typedb_service/analyze.proto"; +import "proto/typedb_service/answer.proto"; +import "proto/typedb_service/options.proto"; +import "proto/typedb_service/error.proto"; package typedb.protocol; diff --git a/proto/server.proto b/proto/typedb_service/server.proto similarity index 91% rename from proto/server.proto rename to proto/typedb_service/server.proto index e29d01e..695a409 100644 --- a/proto/server.proto +++ b/proto/typedb_service/server.proto @@ -14,6 +14,13 @@ message ServerManager { } } + message Get { + message Req {} + message Res { + Server server = 1; + } + } + message Register { message Req { uint64 replica_id = 1; diff --git a/proto/transaction.proto b/proto/typedb_service/transaction.proto similarity index 93% rename from proto/transaction.proto rename to proto/typedb_service/transaction.proto index ce5f648..a47d07b 100644 --- a/proto/transaction.proto +++ b/proto/typedb_service/transaction.proto @@ -4,10 +4,10 @@ syntax = "proto3"; -import "proto/analyze.proto"; -import "proto/error.proto"; -import "proto/options.proto"; -import "proto/query.proto"; +import "proto/typedb_service/analyze.proto"; +import "proto/typedb_service/error.proto"; +import "proto/typedb_service/options.proto"; +import "proto/typedb_service/query.proto"; package typedb.protocol; diff --git a/proto/typedb-service.proto b/proto/typedb_service/typedb_service.proto similarity index 86% rename from proto/typedb-service.proto rename to proto/typedb_service/typedb_service.proto index 3b401c7..03d6648 100644 --- a/proto/typedb-service.proto +++ b/proto/typedb_service/typedb_service.proto @@ -4,13 +4,13 @@ syntax = "proto3"; -import "proto/authentication.proto"; -import "proto/connection.proto"; -import "proto/database.proto"; -import "proto/server.proto"; -import "proto/user.proto"; -import "proto/transaction.proto"; -import "proto/migration.proto"; +import "proto/typedb_service/authentication.proto"; +import "proto/typedb_service/connection.proto"; +import "proto/typedb_service/database.proto"; +import "proto/typedb_service/server.proto"; +import "proto/typedb_service/user.proto"; +import "proto/typedb_service/transaction.proto"; +import "proto/typedb_service/migration.proto"; package typedb.protocol; @@ -24,6 +24,7 @@ service TypeDB { // Server Manager API rpc servers_all (ServerManager.All.Req) returns (ServerManager.All.Res); + rpc servers_get (ServerManager.Get.Req) returns (ServerManager.Get.Res); rpc servers_register(ServerManager.Register.Req) returns (ServerManager.Register.Res); rpc servers_deregister(ServerManager.Deregister.Req) returns (ServerManager.Deregister.Res); diff --git a/proto/user.proto b/proto/typedb_service/user.proto similarity index 100% rename from proto/user.proto rename to proto/typedb_service/user.proto From ec2bbc5d7d86dca69622f8f4e1086bc7961343d5 Mon Sep 17 00:00:00 2001 From: Ganeshwara Hananda Date: Fri, 24 Oct 2025 14:26:58 +0100 Subject: [PATCH 08/16] Add protobuf schema for Raft's 'state mutation request' (#232) Add protobuf schema for Raft's 'state mutation request'. The schema defines how the data is transported over the network and stored in the Raft log. - Add a protobuf definition for State Mutation Request - Create a mock RPC endpoint to handle that message as a hack to get the code generator to work --- grpc/java/BUILD | 6 ++- grpc/nodejs/BUILD | 34 ++++++++----- grpc/rust/BUILD | 3 +- proto/raft_service/BUILD | 7 +++ proto/raft_service/raft_service.proto | 2 + proto/raft_service/request.proto | 72 +++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 proto/raft_service/request.proto diff --git a/grpc/java/BUILD b/grpc/java/BUILD index a793641..b35e96a 100644 --- a/grpc/java/BUILD +++ b/grpc/java/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. package(default_visibility = ["//visibility:public"]) + load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") load("@rules_proto_grpc//java:defs.bzl", "java_grpc_library") @@ -12,8 +13,9 @@ java_grpc_library( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", @@ -36,7 +38,7 @@ java_grpc_library( checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), license_type = "mpl-header", - size = "small", ) diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD index 8e00bae..4db3294 100644 --- a/grpc/nodejs/BUILD +++ b/grpc/nodejs/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. package(default_visibility = ["//visibility:public"]) + load("@typedb_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test") load("@typedb_bazel_distribution//npm:rules.bzl", "assemble_npm", "deploy_npm") load("@typedb_dependencies//distribution:deployment.bzl", "deployment") @@ -14,7 +15,10 @@ load("//grpc/nodejs:rules.bzl", "ts_grpc_compile") npm_link_all_packages(name = "node_modules") -exports_files(["package.json", "pnpm-lock.yaml"]) +exports_files([ + "package.json", + "pnpm-lock.yaml", +]) load("@typedb_protocol_npm//grpc/nodejs:protoc-gen-ts/package_json.bzl", protoc_gen_ts_bin = "bin") @@ -28,8 +32,9 @@ ts_grpc_compile( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", @@ -45,12 +50,14 @@ ts_grpc_compile( "//proto/typedb_service:transaction-proto", "//proto/typedb_service:typedb-service", "//proto/typedb_service:user-proto", - ] + ], ) ts_project( name = "typedb-protocol", srcs = [":typedb-protocol-src"], + declaration = True, + transpiler = "tsc", tsconfig = { "compilerOptions": { "target": "es2019", @@ -60,26 +67,27 @@ ts_project( }, "include": [ "proto", - ] + ], }, - declaration = True, + visibility = ["//visibility:public"], deps = [ ":node_modules/@grpc/grpc-js", - ":node_modules/google-protobuf", ":node_modules/@types/google-protobuf", ":node_modules/@types/node", + ":node_modules/google-protobuf", ":node_modules/typescript", ], - transpiler = "tsc", - visibility = ["//visibility:public"], ) npm_package( name = "typedb-protocol-package", - srcs = [":typedb-protocol", "package.json"], + srcs = [ + "package.json", + ":typedb-protocol", + ], + include_external_repositories = ["typedb_protocol"], include_runfiles = False, package = "typedb-protocol", - include_external_repositories = ["typedb_protocol"], visibility = ["//visibility:public"], ) @@ -90,18 +98,18 @@ assemble_npm( deploy_npm( name = "deploy-npm", - target = ":assemble-npm", - snapshot = deployment["npm"]["snapshot"], release = deployment["npm"]["release"], + snapshot = deployment["npm"]["snapshot"], + target = ":assemble-npm", ) checkstyle_test( name = "checkstyle", + size = "small", include = glob(["*"]), exclude = [ "package.json", "pnpm-lock.yaml", ], license_type = "mpl-header", - size = "small", ) diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD index 200ec89..c3f0595 100644 --- a/grpc/rust/BUILD +++ b/grpc/rust/BUILD @@ -14,8 +14,9 @@ rust_tonic_compile( "//proto:version-proto", "//proto/raft_service:raft-service", "//proto/raft_service:replication-proto", - "//proto/typedb_clustering_service:typedb-clustering-service", + "//proto/raft_service:request-proto", "//proto/typedb_clustering_service:typedb-clustering-proto", + "//proto/typedb_clustering_service:typedb-clustering-service", "//proto/typedb_service:analyze-proto", "//proto/typedb_service:analyzed-conjunction-proto", "//proto/typedb_service:answer-proto", diff --git a/proto/raft_service/BUILD b/proto/raft_service/BUILD index 62000b0..c9fc4f2 100644 --- a/proto/raft_service/BUILD +++ b/proto/raft_service/BUILD @@ -11,6 +11,7 @@ proto_library( srcs = ["raft_service.proto"], deps = [ ":replication-proto", + ":request-proto", ], ) @@ -20,6 +21,12 @@ proto_library( deps = [], ) +proto_library( + name = "request-proto", + srcs = ["request.proto"], + deps = [], +) + checkstyle_test( name = "checkstyle", size = "small", diff --git a/proto/raft_service/raft_service.proto b/proto/raft_service/raft_service.proto index 51886f2..d934687 100644 --- a/proto/raft_service/raft_service.proto +++ b/proto/raft_service/raft_service.proto @@ -5,10 +5,12 @@ syntax = "proto3"; import "proto/raft_service/replication.proto"; +import "proto/raft_service/request.proto"; package typedb.protocol; // TODO: If the protocol is exposed, maybe it's actually `TypeDBRaft`? service Raft { rpc replication (Replication.Req) returns (Replication.Res); + rpc request (Request) returns (Request); } diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto new file mode 100644 index 0000000..2296a36 --- /dev/null +++ b/proto/raft_service/request.proto @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +syntax = "proto3"; + +package typedb.protocol; + +message Request { + oneof request { + Load load = 1; + DatabaseManager database_manager = 2; + DatabaseCommit database_commit = 3; + UserManager user_manager = 4; + } + + message Load {} + + message DatabaseManager { + oneof database_manager { + All all = 1; + Get get = 2; + GetUnrestricted get_unrestricted = 3; + Create create = 4; + CreateUnrestricted create_unrestricted = 5; + Delete delete = 6; + } + + message All {} + message Get { string name = 1; } + message GetUnrestricted { string name = 1; } + message Create { string name = 1; } + message CreateUnrestricted { string name = 1; } + message Delete { string name = 1; } + } + + message DatabaseCommit { + oneof database_commit { + SchemaCommit schema = 1; + DataCommit data = 2; + } + + message SchemaCommit { + string name = 1; + bytes commit_record = 2; + } + + message DataCommit { + string name = 1; + bytes commit_record = 2; + } + } + + message UserManager { + oneof user_manager { + All all = 1; + Get get = 2; + Contains contains = 3; + Create create = 4; + Update update = 5; + Delete delete = 6; + } + + message All { string accessor = 1; } + message Get { string name = 1; string accessor = 2; } + message Contains { string name = 1; } + message Create { bytes commit_record = 1; } + message Update { bytes commit_record = 1; } + message Delete { bytes commit_record = 1; } + } +} + From 47c50a9e2b1c6a19067ec775e9e022b9c6b7cac0 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Wed, 19 Nov 2025 17:49:45 +0000 Subject: [PATCH 09/16] Mirror TypeDB service methods in Raft service (#238) ## Release notes: usage and product changes Reorganize Raft service's request messages to mirror the structure of TypeDB service's methods. Additionally, optionalize the Server message to allow returning pure Raft Agent id without other server information available for Clustered operations (in case the replica is not available). ## Implementation I just did it! --------- Co-authored-by: Ganeshwara Hananda Co-authored-by: Ganeshwara Hananda --- proto/raft_service/request.proto | 105 ++++++++++++++++------ proto/typedb_service/server.proto | 6 +- proto/typedb_service/typedb_service.proto | 1 - 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto index 2296a36..ff88beb 100644 --- a/proto/raft_service/request.proto +++ b/proto/raft_service/request.proto @@ -10,8 +10,9 @@ message Request { oneof request { Load load = 1; DatabaseManager database_manager = 2; - DatabaseCommit database_commit = 3; + Database database = 3; UserManager user_manager = 4; + User user = 5; } message Load {} @@ -19,25 +20,50 @@ message Request { message DatabaseManager { oneof database_manager { All all = 1; - Get get = 2; - GetUnrestricted get_unrestricted = 3; - Create create = 4; - CreateUnrestricted create_unrestricted = 5; - Delete delete = 6; + Contains contains = 2; + Get get = 3; + GetUnrestricted get_unrestricted = 4; + Create create = 5; + CreateUnrestricted create_unrestricted = 6; } message All {} - message Get { string name = 1; } - message GetUnrestricted { string name = 1; } - message Create { string name = 1; } - message CreateUnrestricted { string name = 1; } - message Delete { string name = 1; } + + message Contains { + string name = 1; + } + + message Get { + string name = 1; + } + + message GetUnrestricted { + string name = 1; + } + message Create { + string name = 1; + } + + message CreateUnrestricted { + string name = 1; + } } - message DatabaseCommit { - oneof database_commit { - SchemaCommit schema = 1; - DataCommit data = 2; + message Database { + oneof database { + Schema schema = 1; + TypeSchema type_schema = 2; + SchemaCommit schema_commit = 3; + DataCommit data_commit = 4; + Delete delete = 5; + } + + message Schema { + string name = 1; + } + + message TypeSchema { + string name = 1; } message SchemaCommit { @@ -49,24 +75,51 @@ message Request { string name = 1; bytes commit_record = 2; } + + message Delete { + string name = 1; + } } message UserManager { oneof user_manager { All all = 1; - Get get = 2; - Contains contains = 3; + Contains contains = 2; + Get get = 3; Create create = 4; - Update update = 5; - Delete delete = 6; } - message All { string accessor = 1; } - message Get { string name = 1; string accessor = 2; } - message Contains { string name = 1; } - message Create { bytes commit_record = 1; } - message Update { bytes commit_record = 1; } - message Delete { bytes commit_record = 1; } + message All { + string accessor = 1; + } + + message Contains { + string name = 1; + string accessor = 2; + } + + message Get { + string name = 1; + string accessor = 2; + } + + message Create { + bytes commit_record = 1; + } } -} + message User { + oneof user { + Update update = 1; + Delete delete = 2; + } + + message Update { + bytes commit_record = 1; + } + + message Delete { + bytes commit_record = 1; + } + } +} diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index 695a409..f75f5f3 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -38,13 +38,13 @@ message ServerManager { } message Server { - string address = 1; + optional string address = 1; optional ReplicaStatus replica_status = 2; message ReplicaStatus { uint64 replica_id = 1; - ReplicaType replica_type = 2; - uint64 term = 3; + optional ReplicaType replica_type = 2; + optional uint64 term = 3; enum ReplicaType { Primary = 0; diff --git a/proto/typedb_service/typedb_service.proto b/proto/typedb_service/typedb_service.proto index 03d6648..2dec0f6 100644 --- a/proto/typedb_service/typedb_service.proto +++ b/proto/typedb_service/typedb_service.proto @@ -59,5 +59,4 @@ service TypeDB { // requests and responses back-and-forth. The first transaction client message must // be {Transaction.Open.Req}. Closing the stream closes the transaction. rpc transaction (stream Transaction.Client) returns (stream Transaction.Server); - } From ae75c6edda51a5cccd4b1074bdc3f671ba1da7e2 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Thu, 20 Nov 2025 14:14:32 +0000 Subject: [PATCH 10/16] Rename ReplicaType to ReplicaRole --- proto/typedb_service/server.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index f75f5f3..fa524fa 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -43,10 +43,10 @@ message Server { message ReplicaStatus { uint64 replica_id = 1; - optional ReplicaType replica_type = 2; + optional ReplicaRole replica_role = 2; optional uint64 term = 3; - enum ReplicaType { + enum ReplicaRole { Primary = 0; Candidate = 1; Secondary = 2; From 788f985129629cad413f063288b5dad48927ad49 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Fri, 28 Nov 2025 19:33:40 +0000 Subject: [PATCH 11/16] Complete replication requests for user management (#240) ## Release notes: usage and product changes Introduce protocol changes required by https://github.com/typedb/typedb-cluster/pull/642 : * User management requests now contain specific request information like username and password hashes (not actual passwords) instead of database commits. ## Implementation --- proto/raft_service/request.proto | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/proto/raft_service/request.proto b/proto/raft_service/request.proto index ff88beb..294d24a 100644 --- a/proto/raft_service/request.proto +++ b/proto/raft_service/request.proto @@ -104,7 +104,9 @@ message Request { } message Create { - bytes commit_record = 1; + string name = 1; + string password_hash = 2; + string accessor = 3; } } @@ -115,11 +117,15 @@ message Request { } message Update { - bytes commit_record = 1; + string name = 1; + optional string name_update = 2; + optional string password_hash_update = 3; + string accessor = 4; } message Delete { - bytes commit_record = 1; + string name = 1; + string accessor = 2; } } } From cf8f3cb3db1a9cf41cfc3f3bbb8959246c368e70 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 15 Dec 2025 16:56:44 +0000 Subject: [PATCH 12/16] Return the single target server instead of all servers in the initial connection (#242) ## Release notes: usage and product changes Substitute `ServerManager.All.Res` with `ServerManager.Get.Res` in the result of the connection opening message. Returning all servers in a cloud environment does not make sense, since you cannot know the actual address of other servers until you connect to them (they are probably hidden behind proxies, and the GRPC protocol does not even know about them). So, it makes more sense to return only the target server's server information, which helps map the connection address with the id of the replica and all the other parameters for the future requests. ## Implementation --- proto/typedb_service/connection.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/typedb_service/connection.proto b/proto/typedb_service/connection.proto index d4d3207..5c42a0d 100644 --- a/proto/typedb_service/connection.proto +++ b/proto/typedb_service/connection.proto @@ -24,7 +24,7 @@ message Connection { message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - ServerManager.All.Res servers_all = 3; + ServerManager.Get.Res server = 3; Authentication.Token.Create.Res authentication = 4; } } From 6b83ce686d543f1c73260516adac570c29f8e3d3 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 15 Dec 2025 17:54:54 +0000 Subject: [PATCH 13/16] Update version to 3.7.0-alpha1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 240bba9..f977d07 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.0 \ No newline at end of file +3.7.0-alpha1 \ No newline at end of file From 9dab855e4069e2dd8e481bd57c938ee4d4754b9c Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Tue, 16 Dec 2025 12:29:22 +0000 Subject: [PATCH 14/16] Return servers_all to the connection result --- proto/typedb_service/connection.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/typedb_service/connection.proto b/proto/typedb_service/connection.proto index 5c42a0d..d4d3207 100644 --- a/proto/typedb_service/connection.proto +++ b/proto/typedb_service/connection.proto @@ -24,7 +24,7 @@ message Connection { message Res { uint64 server_duration_millis = 1; ConnectionID connection_id = 2; - ServerManager.Get.Res server = 3; + ServerManager.All.Res servers_all = 3; Authentication.Token.Create.Res authentication = 4; } } From 2ab39b0ff947f598176819affd4132a3169de896 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Tue, 16 Dec 2025 15:19:23 +0000 Subject: [PATCH 15/16] Change version to 3.7.0-alpha0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f977d07..88af471 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.0-alpha1 \ No newline at end of file +3.7.0-alpha0 \ No newline at end of file From 88d31ba3d236dbddb8dea87c1ab2237204d25f73 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Tue, 16 Dec 2025 16:41:27 +0000 Subject: [PATCH 16/16] Add connection address to the Server message --- proto/typedb_service/server.proto | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proto/typedb_service/server.proto b/proto/typedb_service/server.proto index fa524fa..f76792a 100644 --- a/proto/typedb_service/server.proto +++ b/proto/typedb_service/server.proto @@ -38,8 +38,9 @@ message ServerManager { } message Server { - optional string address = 1; - optional ReplicaStatus replica_status = 2; + optional string serving_address = 1; + optional string connection_address = 2; + optional ReplicaStatus replica_status = 3; message ReplicaStatus { uint64 replica_id = 1;