diff --git a/doc/protocol.rst b/doc/protocol.rst index e9d91c5..e3b18a6 100644 --- a/doc/protocol.rst +++ b/doc/protocol.rst @@ -52,12 +52,40 @@ implementation in ZeroMQ, which enforces a strict one request, one response pattern; instead, we use DEALER/ROUTER, which allows any amount of messages in any order, in any direction. +Upon receipt of a request the daemon will immediately issue an ACK response. +The absence of a quick response indicates that the daemon is not available, +and the client should immediately raise an error. After the client receives +the initial ACK it should then look for the full response. There will be no +further messages associated with that id number after the full response is +received. A daemon may choose to forego the ACK response, but should only +do so in circumstances where processing a request requires zero additional +processing time. + +All requests are handled +fully asynchronously; a client could send a thousand requests in quick +succession, but the responses will not be serialized, and the response order +is not guaranteed. Synchronous behavior, if desired, is implemented by client +code and not in the protocol itself. +Here is an example of what the full exchange on the client side might look +like, in this case handling the exchange as a synchronous request:: + + self.socket = zmq_context.socket(zmq.DEALER) + self.socket.setsockopt(zmq.LINGER, 0) + self.socket.identity = identity.encode() + self.socket.connect(daemon) + + self.socket.send_multipart(request) + result = self.socket.poll(100) # milliseconds + if result == 0: + raise TimeoutError('no response received in 100 ms') + + ack = self.socket.recv_multipart() + response = self.socket.recv_multipart() + The request/response interaction between the client and daemon is a multipart -message, where each part is required and has specific meaning. The reference -implementation provides a :class:`mktl.protocol.message.Message` class to -minimize the amount of code that has to be aware about the on-the-wire message -structure. For both ends of the request/response exchange, the message parts -are: +message, where each component of the message has a specific meaning. +The message parts are identical for both ends of the request/response +exchange. .. list-table:: @@ -67,6 +95,7 @@ are: * - **version** - A single ASCII character indicating the mKTL protocol version number. The initial release of the mKTL protocol uses the version character 'a'. + The version identifier is always present. * - **identifier** - A unique identifier for the request. The format of this identifier is @@ -76,73 +105,65 @@ are: to the original request. Note that this identifier does not necessarily have significance on the daemon side, daemons will use their own internal scheme to uniquely identify requests, but the response will always include - this original identifier. + this original identifier. The request identifier is always present. * - **type** - The message type. This is a short string of characters that identifies what type of request, or response, this message represents. It is one - of the values described in the :ref:`message_types` section below. + of the values described in the :ref:`message_types` section below. The + message type is always present. * - **target** - The target for this request/response, if any. Not all requests have a - target; responses don't need to specify it, since it is the identification - number that ties a response to its request. If a target is specified it + target; responses don't need to specify it, since the identifier field + associates a response with a request. If a target is specified it is a store or a key, depending on the request; this field will be an empty byte sequence if the target is not specified. + * - **flags** + - A big-endian integer representing boolean flags that modify how this + message is handled. The default value is an integer zero; if this field + is transmitted as an empty byte sequence it must be interpreted as the + integer zero. Each bit in the integer has a specific meaning: + + .. list-table:: + + * - *Bit* + - *Name* + - *Meaning* + + * - 0b0001 + - NO_ACK + - Suppress the ACK response to this request. + + * - 0b0010 + - NO_REP + - Suppress the REP response to this request. + * - **payload** - The message payload. This is the JSON representation of any additional data required as part of this exchange; if setting a new value, it would contain the value; if it is a response containing additional information - it would go here. This field will be an empty byte sequence if no - additional information is required. See the :ref:`message_payload` section - for a more complete description of the payload contents. + it would go here. See the :ref:`message_payload` section + for a more complete description of the payload contents. This field will + be an empty byte sequence if there is no payload. * - **bulk** - A bulk byte sequence, typically a component of the payload. This is to allow the transmission of information like image data, where the bulk bytes represent the image buffer, and the JSON payload describes how to interpret the buffer. This field will be omitted entirely if there - is no bulk component. - -Upon receipt of a request the daemon will immediately issue an ACK response. -The absence of a quick response indicates that the daemon is not available, -and the client should immediately raise an error. After the client receives -the initial ACK it should then look for the full response. There will be no -further messages associated with that id number after the full response is -received. A daemon may choose to forego the ACK response, but should only -do so in circumstances where processing a request requires zero additional -processing time. - -All requests are handled -fully asynchronously; a client could send a thousand requests in quick -succession, but the responses will not be serialized, and the response order -is not guaranteed. Synchronous behavior, if desired, is implemented by client -code and not in the protocol itself. - -Here is an example of what the full exchange on the client side might look -like, in this case handling the exchange as a synchronous request:: - - self.socket = zmq_context.socket(zmq.DEALER) - self.socket.setsockopt(zmq.LINGER, 0) - self.socket.identity = identity.encode() - self.socket.connect(daemon) - - self.socket.send_multipart(request) - result = self.socket.poll(100) # milliseconds - if result == 0: - raise zmq.ZMQError('no response received in 100 ms') - - ack = self.socket.recv_multipart() - response = self.socket.recv_multipart() + is no bulk component, which allows a recipient to distinguish between + an empty byte sequence and the complete absence of data. Here is a representation of what the on-the-wire messages might look like -for the simple exchange outlined above:: +for a simple GET request:: b'a' b'00000023' b'GET' b'kpfguide.LASTFILENAME' + b'\x00' b'' b'a' @@ -150,13 +171,19 @@ for the simple exchange outlined above:: b'ACK' b'' b'' + b'' b'a' b'00000023' b'REP' b'' + b'' b'{"value": /sdata1701/kpf1/2025-06-23/image_672.fits', "time": 234.23}' +The reference implementation provides a :class:`mktl.protocol.message.Message` +class to minimize the amount of code that has to be aware about the on-the-wire +message structure. + .. _message_types: diff --git a/sbin/mkbrokerd b/sbin/mkbrokerd index e688fe3..f74bea1 100755 --- a/sbin/mkbrokerd +++ b/sbin/mkbrokerd @@ -400,7 +400,8 @@ class RequestServer(mktl.protocol.request.Server): will be generated. """ - self.req_ack(request) + if request.ack: + self.req_ack(request) type = request.type target = request.target @@ -415,7 +416,8 @@ class RequestServer(mktl.protocol.request.Server): else: raise ValueError('invalid request type: ' + type) - return payload + if request.reply: + return payload def req_hash(self, request): diff --git a/src/mktl/daemon.py b/src/mktl/daemon.py index fffebbd..ecd8b54 100644 --- a/src/mktl/daemon.py +++ b/src/mktl/daemon.py @@ -488,11 +488,12 @@ def req_config(self, request): def req_handler(self, request): - """ Inspect the incoming request type and decide how a response - will be generated. + """ Inspect the incoming request type and call an appropriate + method to handle that specific request. """ - self.req_ack(request) + if request.ack: + self.req_ack(request) type = request.type target = request.target @@ -511,7 +512,10 @@ def req_handler(self, request): else: raise ValueError('unhandled request type: ' + type) - return response + if request.reply: + return response + else: + return None def req_get(self, request): diff --git a/src/mktl/item.py b/src/mktl/item.py index 2b4c0d9..5bf3f33 100644 --- a/src/mktl/item.py +++ b/src/mktl/item.py @@ -661,15 +661,18 @@ def req_set(self, request): return payload - def set(self, new_value, wait=True, formatted=False, quantity=False): + def set(self, new_value, wait=True, reply=True, formatted=False, quantity=False): """ Set a new value. Set *wait* to True to block until the request completes; this is the default behavior. If *wait* is set to False, the caller will be returned a :class:`mktl.protocol.message.Request` instance, which has a :func:`mktl.protocol.message.Request.wait` method that can optionally be invoked to block until completion of the request; the wait will return immediately once the request is - satisfied. There is no return value for a blocking request; failed - requests will raise exceptions. + satisfied. Set *reply* to False to disable all error handling and + acknowledgements for the request (fire and forget); setting + *reply to False implies *wait* is also False. + There is no return value for a blocking request; failed requests + will raise exceptions. The optional *formatted* and *quantity* options enable calling :func:`set` with either the string-formatted representation or @@ -699,8 +702,16 @@ def set(self, new_value, wait=True, formatted=False, quantity=False): raise ValueError('formatted+quantity arguments must be boolean') payload = self.to_payload(new_value) - payload.add_origin() - message = protocol.message.Request('SET', self.full_key, payload) + + if reply: + flags = None + payload.add_origin() + else: + flags = protocol.message.NO_ACK_OR_REP + wait = False + + key = self.full_key + message = protocol.message.Request('SET', key, payload, flags=flags) self.req.send(message) if wait == False: @@ -841,7 +852,7 @@ def to_format(self, value): return formatted - def to_payload(self, value=None, timestamp=None): + def to_payload(self, value=None, timestamp=None, **kwargs): """ Interpret the provided arguments into a :class:`mktl.protocol.message.Payload` instance; if the *value* is not specified the current value of this :class:`Item` will be @@ -877,11 +888,11 @@ def to_payload(self, value=None, timestamp=None): bulk = value.tobytes() except AttributeError: bulk = None - payload = protocol.message.Payload(value, timestamp) + payload = protocol.message.Payload(value, timestamp, **kwargs) else: shape = value.shape dtype = str(value.dtype) - payload = protocol.message.Payload(None, timestamp, bulk=bulk, shape=shape, dtype=dtype) + payload = protocol.message.Payload(None, timestamp, bulk=bulk, shape=shape, dtype=dtype, **kwargs) return payload diff --git a/src/mktl/protocol/message.py b/src/mktl/protocol/message.py index 9027485..19015f2 100644 --- a/src/mktl/protocol/message.py +++ b/src/mktl/protocol/message.py @@ -19,6 +19,12 @@ version = b'a' +# Define any/all optional flags for message handling. + +NO_ACK = 0b1 +NO_REP = 0b10 +NO_ACK_OR_REP = NO_ACK | NO_REP + # The cached origin information is used by the Payload class to (optionally) # provide information used to determine the origin of a message. The call to # os.getlogin() appears to be more expensive than the others. For that reason @@ -77,7 +83,7 @@ class Message: valid_types = set(('ACK', 'REP')) - def __init__(self, type, target=None, payload=None, id=None): + def __init__(self, type, target=None, payload=None, id=None, flags=None): if type in self.valid_types: pass @@ -88,6 +94,7 @@ def __init__(self, type, target=None, payload=None, id=None): # for example, publish messages do not have or need an identification # number or a prefix. + self.flags = flags self.id = id self.type = type self.payload = payload @@ -108,6 +115,15 @@ def __repr__(self): return repr(self._parts) + @property + def ack(self): + + if self.flags and self.flags & NO_ACK: + return False + else: + return True + + def _finalize(self): """ Take the contents of this :class:`Message`, interpet them as bytes, and prepare the tuple that will be used for the multipart @@ -118,6 +134,7 @@ def _finalize(self): # Once finalized, always finalized. return + flags = self.flags id = self.id type = self.type target = self.target @@ -147,6 +164,11 @@ def _finalize(self): # Assume it is already bytes. pass + if flags: + flags = flags.to_bytes(byteorder='big') + else: + flags = b'' + if payload is None or payload == '': bulk = None payload = b'' @@ -158,9 +180,9 @@ def _finalize(self): # be represented as None, distinct from being an empty byte sequence. if bulk is None: - parts = (version, id, type, target, payload) + parts = (version, id, type, target, flags, payload) else: - parts = (version, id, type, target, payload, bulk) + parts = (version, id, type, target, flags, payload, bulk) if self.prefix: parts = self.prefix + parts @@ -208,27 +230,33 @@ def reconstruct(cls, parts): quantity = len(parts) - if quantity != 5 and quantity != 6: - raise ValueError("expected 5 or 6 parts, got %d" % (quantity)) + if quantity != 6 and quantity != 7: + raise ValueError("expected 6 or 7 parts, got %d" % (quantity)) their_version = parts[0] if their_version != version: raise ValueError("version mismatch: expected %s, got %s" % (repr(version), repr(their_version))) - message_id = parts[1] - message_type = parts[2] + id = parts[1] + type = parts[2] target = parts[3] - payload = parts[4] + flags = parts[4] + payload = parts[5] try: - bulk = parts[5] + bulk = parts[6] except IndexError: bulk = None - message_type = message_type.decode() + type = type.decode() target = target.decode() + if flags == b'': + flags = None + else: + flags = int.from_bytes(flags, byteorder='big') + if payload == b'': payload = None else: @@ -240,10 +268,19 @@ def reconstruct(cls, parts): # allow it to pass, assuming the users know what they're doing. pass - message = cls(message_type, target, payload, message_id) + message = cls(type, target, payload, id, flags) return message + @property + def reply(self): + + if self.flags and self.flags & NO_REP: + return False + else: + return True + + # end of class Message @@ -346,7 +383,7 @@ class Request(Message): valid_types = set(('CONFIG', 'GET', 'HASH', 'SET')) - def __init__(self, type, target=None, payload=None, id=None): + def __init__(self, type, target=None, payload=None, id=None, flags=None): # Requests are generally initiated without an id number, but they're # required to have one. The expectation is that requests will have an @@ -361,7 +398,7 @@ def __init__(self, type, target=None, payload=None, id=None): if id is None: id = _id_next() - Message.__init__(self, type, target, payload, id) + Message.__init__(self, type, target, payload, id, flags) self.response = None @@ -500,6 +537,20 @@ def __repr__(self): return self.encapsulate().decode() + def add_origin(self): + """ Add fields to this payload to provide information describing + the origin of this message. The primary use case is for debugging + or logging, as opposed to uniquely identifying the sender. + """ + + self._user = _origin_user + self._hostname = _origin_hostname + self._pid = _origin_pid + self._ppid = _origin_ppid + self._executable = sys.executable + self._argv = sys.argv + + def encapsulate(self): """ Add all non-omitted local attributes to a dictionary, and return the JSON encoding of that dictionary. For example, if the .value @@ -537,20 +588,6 @@ def encapsulate(self): return payload - def add_origin(self): - """ Add fields to this payload to provide information describing - the origin of this message. The primary use case is for debugging - or logging, as opposed to uniquely identifying the sender. - """ - - self._user = _origin_user - self._hostname = _origin_hostname - self._pid = _origin_pid - self._ppid = _origin_ppid - self._executable = sys.executable - self._argv = sys.argv - - # end of class Payload diff --git a/src/mktl/protocol/request.py b/src/mktl/protocol/request.py index 23acd5a..23d0535 100644 --- a/src/mktl/protocol/request.py +++ b/src/mktl/protocol/request.py @@ -40,6 +40,7 @@ def __init__(self, address, port): self.socket = zmq_context.socket(zmq.DEALER) self.socket.setsockopt(zmq.LINGER, 0) + self.socket.set_hwm(0) self.socket.identity = identity.encode() self.socket.connect(server) @@ -181,6 +182,11 @@ def send(self, message): self.requests.put(message) self.request_signal.send(b'') + if message.ack: + pass + else: + return + ack = message.wait_ack(self.timeout) if ack == False: @@ -222,6 +228,7 @@ def __init__(self, hostname=None, port=None, avoid=set()): self.hostname = hostname self.socket = zmq_context.socket(zmq.ROUTER) self.socket.setsockopt(zmq.LINGER, 0) + self.socket.set_hwm(0) # If the port is set, use it; otherwise, look for the first available # port within the default range. @@ -340,7 +347,14 @@ def req_handler(self, request): structure of what's happening in the daemon code. """ - self.req_ack(request) + + if request.ack: + self.req_ack(request) + + if request.reply: + pass + else: + return response = message.Message('REP', target, id=request.id) response.prefix = request.prefix @@ -391,7 +405,8 @@ def req_incoming(self, parts): if payload is None and error is None: # The handler should only return None when no response is # immediately forthcoming-- the handler has invoked some - # other processing chain that will issue a proper response. + # other processing chain that will issue a proper response, + # or the client explicitly requested no response. return if error is not None: @@ -423,6 +438,9 @@ def run(self): elif self.socket == active: parts = self.socket.recv_multipart() # Calling submit() will block if a worker is not available. + # Note that for high frequency operations this can result + # in out-of-order handling of requests, for example, if a + # stream of SET requests are inbound for a single item. self.workers.submit(self.req_incoming, parts)