Divided protobuf messages into their respective domains. Added support for google.protobuf message types. Fixes (#279)

Author: tegefaulkes

src/agent/GRPCClientAgent.ts

@@ -1,7 +1,7 @@
 import type { TLSConfig } from '../network/types';
 
 import { GRPCClient, utils as grpcUtils } from '../grpc';
-import * as agentPB from '../proto/js/Agent_pb';
+import { messages } from '.';
 import { AgentClient } from '../proto/js/Agent_grpc_pb';
 import { NodeId } from '../nodes/types';
 import { Host, Port, ProxyConfig } from '../network/types';
@@ -63,15 +63,15 @@ class GRPCClientAgent extends GRPCClient<AgentClient> {
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public echo(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.EchoMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.common.EchoMessage>(
       this.client,
       this.client.echo,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public vaultsGitInfoGet(...args) {
-    return grpcUtils.promisifyReadableStreamCall<agentPB.PackChunk>(
+    return grpcUtils.promisifyReadableStreamCall<messages.vaults.PackChunk>(
       this.client,
       this.client.vaultsGitInfoGet,
     )(...args);
@@ -84,55 +84,55 @@ class GRPCClientAgent extends GRPCClient<AgentClient> {
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public vaultsScan(...args) {
-    return grpcUtils.promisifyReadableStreamCall<agentPB.VaultListMessage>(
+    return grpcUtils.promisifyReadableStreamCall<messages.vaults.Vault>(
       this.client,
       this.client.vaultsScan,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public nodesClosestLocalNodesGet(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.NodeTableMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.nodes.NodeTable>(
       this.client,
       this.client.nodesClosestLocalNodesGet,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public nodesClaimsGet(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.ClaimsMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.nodes.Claims>(
       this.client,
       this.client.nodesClaimsGet,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public nodesChainDataGet(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.ChainDataMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.nodes.ChainData>(
       this.client,
       this.client.nodesChainDataGet,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public nodesHolePunchMessageSend(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.EmptyMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.common.EmptyMessage>(
       this.client,
       this.client.nodesHolePunchMessageSend,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public notificationsSend(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.NotificationMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.notifications.AgentNotification>(
       this.client,
       this.client.notificationsSend,
     )(...args);
   }
 
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public vaultsPermisssionsCheck(...args) {
-    return grpcUtils.promisifyUnaryCall<agentPB.PermissionMessage>(
+    return grpcUtils.promisifyUnaryCall<messages.vaults.NodePermissionAllowed>(
       this.client,
       this.client.vaultsPermisssionsCheck,
     )(...args);
@@ -141,8 +141,8 @@ class GRPCClientAgent extends GRPCClient<AgentClient> {
   @ready(new grpcErrors.ErrorGRPCClientNotStarted())
   public nodesCrossSignClaim(...args) {
     return grpcUtils.promisifyDuplexStreamCall<
-      agentPB.CrossSignMessage,
-      agentPB.CrossSignMessage
+      messages.nodes.CrossSign,
+      messages.nodes.CrossSign
     >(
       this.client,
       this.client.nodesCrossSignClaim,

src/agent/agentPB.ts

@@ -0,0 +1,4 @@
+export * as common from '../proto/js/Common_pb';
+export * as vaults from '../proto/js/Vaults_pb';
+export * as nodes from '../proto/js/Nodes_pb';
+export * as notifications from '../proto/js/Notifications_pb';

src/agent/agentService.ts

@@ -16,7 +16,7 @@ import { NotificationsManager } from '../notifications';
 import { ErrorGRPC } from '../grpc/errors';
 import { AgentService, IAgentServer } from '../proto/js/Agent_grpc_pb';
 
-import * as agentPB from '../proto/js/Agent_pb';
+import { messages } from '.';
 import * as grpcUtils from '../grpc/utils';
 import {
   utils as notificationsUtils,
@@ -48,19 +48,25 @@ function createAgentService({
 }): IAgentServer {
   const agentService: IAgentServer = {
     echo: async (
-      call: grpc.ServerUnaryCall<agentPB.EchoMessage, agentPB.EchoMessage>,
-      callback: grpc.sendUnaryData<agentPB.EchoMessage>,
+      call: grpc.ServerUnaryCall<
+        messages.common.EchoMessage,
+        messages.common.EchoMessage
+      >,
+      callback: grpc.sendUnaryData<messages.common.EchoMessage>,
     ): Promise<void> => {
-      const response = new agentPB.EchoMessage();
+      const response = new messages.common.EchoMessage();
       response.setChallenge(call.request.getChallenge());
       callback(null, response);
     },
     vaultsGitInfoGet: async (
-      call: grpc.ServerWritableStream<agentPB.InfoRequest, agentPB.PackChunk>,
+      call: grpc.ServerWritableStream<
+        messages.vaults.Vault,
+        messages.vaults.PackChunk
+      >,
     ): Promise<void> => {
       const genWritable = grpcUtils.generatorWritable(call);
       const request = call.request;
-      const vaultNameOrId = request.getVaultId();
+      const vaultNameOrId = request.getNameOrId();
       let vaultId, vaultName;
       try {
         vaultId = makeVaultId(idUtils.fromString(vaultNameOrId));
@@ -80,7 +86,7 @@ function createAgentService({
       meta.set('vaultName', vaultName);
       meta.set('vaultId', makeVaultIdPretty(vaultId));
       genWritable.stream.sendMetadata(meta);
-      const response = new agentPB.PackChunk();
+      const response = new messages.vaults.PackChunk();
       const responseGen = vaultManager.handleInfoRequest(vaultId);
       for await (const byte of responseGen) {
         if (byte !== null) {
@@ -93,7 +99,10 @@ function createAgentService({
       await genWritable.next(null);
     },
     vaultsGitPackGet: async (
-      call: grpc.ServerDuplexStream<agentPB.PackChunk, agentPB.PackChunk>,
+      call: grpc.ServerDuplexStream<
+        messages.vaults.PackChunk,
+        messages.vaults.PackChunk
+      >,
     ) => {
       const write = promisify(call.write).bind(call);
       const clientBodyBuffers: Buffer[] = [];
@@ -123,7 +132,7 @@ function createAgentService({
           }
         }
         // TODO: Check the permissions here
-        const response = new agentPB.PackChunk();
+        const response = new messages.vaults.PackChunk();
         const [sideBand, progressStream] = await vaultManager.handlePackRequest(
           vaultId,
           Buffer.from(body),
@@ -151,12 +160,12 @@ function createAgentService({
     },
     vaultsScan: async (
       call: grpc.ServerWritableStream<
-        agentPB.NodeIdMessage,
-        agentPB.VaultListMessage
+        messages.nodes.Node,
+        messages.vaults.Vault
       >,
     ): Promise<void> => {
       const genWritable = grpcUtils.generatorWritable(call);
-      const response = new agentPB.VaultListMessage();
+      const response = new messages.vaults.Vault();
       const id = makeNodeId(call.request.getNodeId());
       try {
         throw Error('Not implemented');
@@ -165,7 +174,7 @@ function createAgentService({
         let listResponse;
         for await (const vault of listResponse) {
           if (vault !== null) {
-            response.setVault(vault);
+            response.setNameOrId(vault);
             await genWritable.next(response);
           } else {
             await genWritable.next(null);
@@ -183,22 +192,19 @@ function createAgentService({
      * @param callback
      */
     nodesClosestLocalNodesGet: async (
-      call: grpc.ServerUnaryCall<
-        agentPB.NodeIdMessage,
-        agentPB.NodeTableMessage
-      >,
-      callback: grpc.sendUnaryData<agentPB.NodeTableMessage>,
+      call: grpc.ServerUnaryCall<messages.nodes.Node, messages.nodes.NodeTable>,
+      callback: grpc.sendUnaryData<messages.nodes.NodeTable>,
     ): Promise<void> => {
-      const response = new agentPB.NodeTableMessage();
+      const response = new messages.nodes.NodeTable();
       try {
         const targetNodeId = makeNodeId(call.request.getNodeId());
         // Get all local nodes that are closest to the target node from the request
         const closestNodes = await nodeManager.getClosestLocalNodes(
           targetNodeId,
         );
         for (const node of closestNodes) {
-          const addressMessage = new agentPB.NodeAddressMessage();
-          addressMessage.setIp(node.address.ip);
+          const addressMessage = new messages.nodes.Address();
+          addressMessage.setHost(node.address.ip);
           addressMessage.setPort(node.address.port);
           // Add the node to the response's map (mapping of node ID -> node address)
           response.getNodeTableMap().set(node.id, addressMessage);
@@ -215,12 +221,12 @@ function createAgentService({
      */
     nodesClaimsGet: async (
       call: grpc.ServerUnaryCall<
-        agentPB.ClaimTypeMessage,
-        agentPB.ClaimsMessage
+        messages.nodes.ClaimType,
+        messages.nodes.Claims
       >,
-      callback: grpc.sendUnaryData<agentPB.ClaimsMessage>,
+      callback: grpc.sendUnaryData<messages.nodes.Claims>,
     ): Promise<void> => {
-      const response = new agentPB.ClaimsMessage();
+      const response = new messages.nodes.Claims();
       // Response.setClaimsList(
       //   await sigchain.getClaims(call.request.getClaimtype() as ClaimType)
       // );
@@ -231,24 +237,24 @@ function createAgentService({
      */
     nodesChainDataGet: async (
       call: grpc.ServerUnaryCall<
-        agentPB.EmptyMessage,
-        agentPB.ChainDataMessage
+        messages.common.EmptyMessage,
+        messages.nodes.ChainData
       >,
-      callback: grpc.sendUnaryData<agentPB.ChainDataMessage>,
+      callback: grpc.sendUnaryData<messages.nodes.ChainData>,
     ): Promise<void> => {
-      const response = new agentPB.ChainDataMessage();
+      const response = new messages.nodes.ChainData();
       try {
         const chainData = await nodeManager.getChainData();
         // Iterate through each claim in the chain, and serialize for transport
         for (const c in chainData) {
           const claimId = c as ClaimIdString;
           const claim = chainData[claimId];
-          const claimMessage = new agentPB.ClaimMessage();
+          const claimMessage = new messages.nodes.AgentClaim();
           // Will always have a payload (never undefined) so cast as string
           claimMessage.setPayload(claim.payload as string);
           // Add the signatures
           for (const signatureData of claim.signatures) {
-            const signature = new agentPB.SignatureMessage();
+            const signature = new messages.nodes.Signature();
             // Will always have a protected header (never undefined) so cast as string
             signature.setProtected(signatureData.protected as string);
             signature.setSignature(signatureData.signature);
@@ -263,10 +269,13 @@ function createAgentService({
       callback(null, response);
     },
     nodesHolePunchMessageSend: async (
-      call: grpc.ServerUnaryCall<agentPB.RelayMessage, agentPB.EmptyMessage>,
-      callback: grpc.sendUnaryData<agentPB.EmptyMessage>,
+      call: grpc.ServerUnaryCall<
+        messages.nodes.Relay,
+        messages.common.EmptyMessage
+      >,
+      callback: grpc.sendUnaryData<messages.common.EmptyMessage>,
     ): Promise<void> => {
-      const response = new agentPB.EmptyMessage();
+      const response = new messages.common.EmptyMessage();
       try {
         // Firstly, check if this node is the desired node
         // If so, then we want to make this node start sending hole punching packets
@@ -292,12 +301,12 @@ function createAgentService({
     },
     notificationsSend: async (
       call: grpc.ServerUnaryCall<
-        agentPB.NotificationMessage,
-        agentPB.EmptyMessage
+        messages.notifications.AgentNotification,
+        messages.common.EmptyMessage
       >,
-      callback: grpc.sendUnaryData<agentPB.EmptyMessage>,
+      callback: grpc.sendUnaryData<messages.common.EmptyMessage>,
     ): Promise<void> => {
-      const response = new agentPB.EmptyMessage();
+      const response = new messages.common.EmptyMessage();
       try {
         const jwt = call.request.getContent();
         const notification = await notificationsUtils.verifyAndDecodeNotif(jwt);
@@ -313,12 +322,12 @@ function createAgentService({
     },
     vaultsPermisssionsCheck: async (
       call: grpc.ServerUnaryCall<
-        agentPB.VaultPermMessage,
-        agentPB.PermissionMessage
+        messages.vaults.NodePermission,
+        messages.vaults.NodePermissionAllowed
       >,
-      callback: grpc.sendUnaryData<agentPB.PermissionMessage>,
+      callback: grpc.sendUnaryData<messages.vaults.NodePermissionAllowed>,
     ): Promise<void> => {
-      const response = new agentPB.PermissionMessage();
+      const response = new messages.vaults.NodePermissionAllowed();
       try {
         const nodeId = makeNodeId(call.request.getNodeId());
         const vaultId = makeVaultId(call.request.getVaultId());
@@ -340,8 +349,8 @@ function createAgentService({
     },
     nodesCrossSignClaim: async (
       call: grpc.ServerDuplexStream<
-        agentPB.CrossSignMessage,
-        agentPB.CrossSignMessage
+        messages.nodes.CrossSign,
+        messages.nodes.CrossSign
       >,
     ) => {
       // TODO: Move all "await genClaims.throw" to a final catch(). Wrap this

src/agent/index.ts

@@ -1,4 +1,4 @@
 export { default as createAgentService, AgentService } from './agentService';
 export { default as GRPCClientAgent } from './GRPCClientAgent';
 export * as errors from './errors';
-export * as agentPB from '../proto/js/Agent_pb';
+export * as messages from './agentPB';

src/bin/agent/lockall.ts

@@ -5,7 +5,7 @@ import * as grpcErrors from '../../grpc/errors';
 import PolykeyClient from '../../PolykeyClient';
 import Logger, { LogLevel, StreamHandler } from '@matrixai/logger';
 
-import { clientPB } from '../../client';
+import { messages } from '../../client';
 
 const lockall = binUtils.createCommand('lockall', {
   description:
@@ -29,7 +29,7 @@ lockall.action(async (options) => {
   clientConfig['nodePath'] = nodePath;
 
   const client = await PolykeyClient.createPolykeyClient(clientConfig);
-  const m = new clientPB.EmptyMessage();
+  const m = new messages.common.EmptyMessage();
 
   try {
     await client.start({});

src/bin/agent/stop.ts

@@ -1,5 +1,5 @@
 import Logger, { LogLevel, StreamHandler } from '@matrixai/logger';
-import { clientPB } from '../../client';
+import { messages } from '../../client';
 import PolykeyClient from '../../PolykeyClient';
 import * as utils from '../../utils';
 import * as binUtils from '../utils';
@@ -27,7 +27,7 @@ stop.action(async (options) => {
     : utils.getDefaultNodePath();
 
   const client = await PolykeyClient.createPolykeyClient(clientConfig);
-  const emptyMessage = new clientPB.EmptyMessage();
+  const emptyMessage = new messages.common.EmptyMessage();
 
   try {
     await client.start({});

src/bin/agent/unlock.ts

@@ -7,7 +7,7 @@ import * as grpcErrors from '../../grpc/errors';
 import PolykeyClient from '../../PolykeyClient';
 import Logger, { LogLevel, StreamHandler } from '@matrixai/logger';
 
-import { clientPB } from '../../client';
+import { messages } from '../../client';
 
 const unlock = binUtils.createCommand('unlock', {
   description:
@@ -19,7 +19,7 @@ const unlock = binUtils.createCommand('unlock', {
 });
 unlock.arguments('[password]');
 unlock.action(async (password, options) => {
-  const sessionPasswordMessage = new clientPB.PasswordMessage();
+  const sessionPasswordMessage = new messages.sessions.Password();
 
   const clientConfig = {};
   clientConfig['logger'] = new Logger('CLI Logger', LogLevel.WARN, [