From b76c4e8577f108858d66ce14f8697908f9611add Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 11:37:54 +0000 Subject: [PATCH 01/46] add hardware receive and transmit handlers --- hopper/server/handler.c | 1 + hopper/server/handler.h | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hopper/server/handler.c b/hopper/server/handler.c index fe312e9..babfa26 100644 --- a/hopper/server/handler.c +++ b/hopper/server/handler.c @@ -18,6 +18,7 @@ static struct HandlerMapping HANDLER_MAP[] = { #endif HANDLER_START_BUTTON, HANDLER_STARTER, HANDLER_HARDWARE, + HANDLER_HARDWARE_RX, HANDLER_HARDWARE_TX, }; /// Maps a handler string to an ID number diff --git a/hopper/server/handler.h b/hopper/server/handler.h index 1f307bb..dac8ae8 100644 --- a/hopper/server/handler.h +++ b/hopper/server/handler.h @@ -15,10 +15,11 @@ #define HANDLER_START_BUTTON {5, "start-button"} #define HANDLER_STARTER {6, "starter"} #define HANDLER_HARDWARE {7, "hardware"} +#define HANDLER_HARDWARE_RX {8, "hardware-rx"} +#define HANDLER_HARDWARE_TX {9, "hardware-tx"} -#define MAX_HANDLER_ID 7 +#define MAX_HANDLER_ID 9 short map_handler_to_id(char *handler); - #endif // handler_h_INCLUDED From bddd1f27a28eaf7a23f8b2aa7686eb40a30ce2f5 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 11:42:02 +0000 Subject: [PATCH 02/46] purge existing hopper sources --- .gitignore | 1 + .vscode/launch.json | 15 -- build.sh | 21 -- hopper/client/__init__.py | 4 - hopper/client/client.py | 57 ----- hopper/client/reader.py | 69 ----- hopper/common/__init__.py | 5 - hopper/common/pipe.py | 143 ----------- hopper/common/pipe_name.py | 59 ----- hopper/common/pipe_type.py | 3 - hopper/server/handler.c | 35 --- hopper/server/handler.h | 25 -- hopper/server/pipe.c | 241 ------------------ hopper/server/pipe.h | 42 ---- hopper/server/server.c | 499 ------------------------------------- hopper/server/server.h | 35 --- hopper/util/read.py | 17 -- hopper/util/write.py | 18 -- setup.py | 10 - 19 files changed, 1 insertion(+), 1298 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100755 build.sh delete mode 100644 hopper/client/__init__.py delete mode 100644 hopper/client/client.py delete mode 100644 hopper/client/reader.py delete mode 100644 hopper/common/__init__.py delete mode 100644 hopper/common/pipe.py delete mode 100644 hopper/common/pipe_name.py delete mode 100644 hopper/common/pipe_type.py delete mode 100644 hopper/server/handler.c delete mode 100644 hopper/server/handler.h delete mode 100644 hopper/server/pipe.c delete mode 100644 hopper/server/pipe.h delete mode 100644 hopper/server/server.c delete mode 100644 hopper/server/server.h delete mode 100755 hopper/util/read.py delete mode 100755 hopper/util/write.py delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index adce25a..e98c1f0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build/ hopper.egg-info/ tmplog.txt .gdb_history +.vscode/ diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 72f09a5..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Run Hopper Server", - "type": "debugpy", - "request": "launch", - "module": "hopper.server", - "args": ["pipes"] - } - ] -} \ No newline at end of file diff --git a/build.sh b/build.sh deleted file mode 100755 index b22c3d1..0000000 --- a/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e - -CC=gcc - -echo "[*] Building Hopper server..." - -mkdir -p build/ -$CC -o build/hopper.server hopper/server/server.c hopper/server/pipe.c hopper/server/handler.c -Ihopper/server -Wall -Wextra -g - -echo "[*] Installing Hopper server..." - -hopper_server=$(realpath build/hopper.server) -ln -s "$hopper_server" /usr/bin/hopper.server - -echo "[*] Installing Hopper client..." - -pip install -e . - -echo "[✓] Done." diff --git a/hopper/client/__init__.py b/hopper/client/__init__.py deleted file mode 100644 index d60a978..0000000 --- a/hopper/client/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .client import HopperClient -from .reader import * - -__all__ = ["HopperClient", "PipeReader", "JsonReader"] diff --git a/hopper/client/client.py b/hopper/client/client.py deleted file mode 100644 index ee83130..0000000 --- a/hopper/client/client.py +++ /dev/null @@ -1,57 +0,0 @@ -from hopper.common import * - - -class HopperClient: - def __init__(self): - """ - Initialize a new HopperClient. - """ - self.__pipes = [] - - def open_pipe(self, pn, create=True, delete=False, blocking=False, use_read_buffer=False, read_buffer_terminator=b'\n'): - """ - Open a pipe specified by a PipeName `pn`. - """ - pipe = Pipe(pn, create=create, delete=delete, blocking=blocking, - use_read_buffer=use_read_buffer, read_buffer_terminator=read_buffer_terminator) - self.__pipes.append(pipe) - - def close_pipe(self, pn): - """ - Close a pipe specified by a PipeName `pn`. - """ - p = self.get_pipe_by_pipe_name(pn) - if p == None: - raise - - p.close() - self.__pipes.remove(p) - - def get_pipe_by_pipe_name(self, pn): - """ - Return the Pipe object specified by PipeName `pn`. - """ - for p in self.__pipes: - if p.pipe_name == pn: - return p - return None - - def read(self, pn, _buf_size=-1): - """ - Read content from the pipe specified by `pn`. - """ - p = self.get_pipe_by_pipe_name(pn) - if p == None: - raise - - return p.read(_buf_size=_buf_size) - - def write(self, pn, buf): - """ - Write `buf` to the PipeName specified by `pn`. - """ - p = self.get_pipe_by_pipe_name(pn) - if p == None: - raise - - return p.write(buf + b'\n') diff --git a/hopper/client/reader.py b/hopper/client/reader.py deleted file mode 100644 index 54b3b1d..0000000 --- a/hopper/client/reader.py +++ /dev/null @@ -1,69 +0,0 @@ -import json - - -class PipeReader: - def __init__(self, client, pipe_name): - self._HOPPER_CLIENT = client - self._PIPE_NAME = pipe_name - - def read(self): - return self._HOPPER_CLIENT.read(self._PIPE_NAME) - - -class JsonReader(PipeReader): - def __init__(self, client, pipe_name, read_validator=None): - super().__init__(client, pipe_name) - - self.read_validator = read_validator if read_validator else self.default_read_validator - self.tail = "" - - if not self._HOPPER_CLIENT.get_pipe_by_pipe_name(pipe_name).blocking: - print("WARN: Non-blocking reads may crash the brain!") - - @staticmethod - def default_read_validator(_): - return True - - def _try_decode_json(self, s): - decoder = json.JSONDecoder() - idx = 0 - while idx < len(s): - slice = s[idx:].lstrip() - if not slice: - break - offset = len(s[idx:]) - len(slice) - try: - obj, end = decoder.raw_decode(slice) - if self.read_validator(obj): - idx += offset + end - return obj, s[idx:].rstrip() - else: - idx += 1 - except json.JSONDecodeError as e: - idx += 1 - return None, s - - def read(self): - buffer = self.tail - chunk_size = 1024 - max_buffer_size = 1024 * 1024 - - while len(buffer) < max_buffer_size: - try: - chunk = self._HOPPER_CLIENT.read( - self._PIPE_NAME, _buf_size=chunk_size) - if not chunk: - break - - buffer += chunk.decode("utf-8") - - obj, tail = self._try_decode_json(buffer) - - if obj: - self.tail = tail - return obj - - except BlockingIOError: - continue - - return None diff --git a/hopper/common/__init__.py b/hopper/common/__init__.py deleted file mode 100644 index 5e9290f..0000000 --- a/hopper/common/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .pipe import Pipe -from .pipe_type import PipeType -from .pipe_name import PipeName - -__all__ = ["PipeType", "PipeName", "Pipe"] \ No newline at end of file diff --git a/hopper/common/pipe.py b/hopper/common/pipe.py deleted file mode 100644 index 63c3552..0000000 --- a/hopper/common/pipe.py +++ /dev/null @@ -1,143 +0,0 @@ -import os - -from .pipe_name import PipeName - -""" -Pipe class: - Represents a single input/output pipe. - Implements basic read/write operations. -""" - - -class Pipe: - __BUF_SIZE = 1024 - __fd = 0 - __pn = None - __handler = None - - def __init__(self, pn: PipeName, create=False, delete=False, blocking=False, use_read_buffer=False, read_buffer_terminator=b'\n', buffer_size=1024): - self.__pn = pn - self.__create = create - self.__delete = delete - self.__blocking = blocking - self.__use_read_buffer = use_read_buffer - self.__read_buffer_terminator = read_buffer_terminator - self.__BUF_SIZE = buffer_size - - # The read buffer with non-blocking I/O, blocks, but not nicely - if not self.__blocking and self.__use_read_buffer: - print("WARN: Buffered reads with non-blocking I/O can crash the brain!") - - self.__open() - - def __open(self): - if self.__pn == None: - return - - pipe_path = self.__pn.pipe_path - - if not os.path.exists(pipe_path) and self.__create: - os.mkfifo(pipe_path) - elif not os.path.exists(pipe_path): - raise FileNotFoundError(f"Cannot find pipe at '{pipe_path}'.") - - flags = os.O_RDWR - flags |= (0 if self.__blocking else os.O_NONBLOCK) - - self.__fd = os.open(pipe_path, flags) - self.__inode_number = os.stat(pipe_path).st_ino - - def read(self, _buf_size=-1): - if self.__use_read_buffer: - buf = b'' - # Consume greedily, until we reach the terminator - while True: - try: - b = os.read(self.__fd, 1) - buf += b - if b == self.__read_buffer_terminator: - break - except BlockingIOError: - continue - except: - return None - return buf - - try: - buf = os.read( - self.__fd, self.__BUF_SIZE if _buf_size == -1 else _buf_size) - except: - return None - return buf - - def write(self, buf): - try: - os.write(self.__fd, buf) - except: - raise - - def close(self): - if self.__pn == None: - return - - os.close(self.__fd) - if self.__delete: - os.remove(self.__pn.pipe_path) - - def __del__(self): - self.close() - - def set_handler(self, handlers): - if self.__pn == None: - raise ValueError("Bad pipe name") - - try: - self.__handler = handlers[self.__pn.handler_id] - except: - raise - - @property - def type(self): - if self.__pn == None: - return None - return self.__pn.type - - @property - def id(self): - if self.__pn == None: - return None - return self.__pn.id - - @property - def handler_id(self): - if self.__pn == None: - return None - return self.__pn.handler_id - - @property - def pipe_name(self): - if self.__pn == None: - return None - return self.__pn - - @property - def pipe_path(self): - if self.__pn == None: - return None - return self.__pn.pipe_path - - @property - def handler(self): - return self.__handler - - @property - def fd(self): - return self.__fd - - @property - def inode_number(self): - return self.__inode_number - - @property - def blocking(self): - return self.__blocking diff --git a/hopper/common/pipe_name.py b/hopper/common/pipe_name.py deleted file mode 100644 index cfdaf69..0000000 --- a/hopper/common/pipe_name.py +++ /dev/null @@ -1,59 +0,0 @@ -import os - -from .pipe_type import PipeType - -class PipeName: - __type = None - __handler_id = None - __id = None - __root = "" - - def __init__(self, x, root = ""): - self.__root = root - - if type(x) is str: - self.__from_str(x) - elif type(x) is tuple: - self.__from_tuple(x) - else: - raise ValueError("Invalid pipe name object.") - - def __from_str(self, s): - p = s.split("_") - self.__type = (PipeType.INPUT if p[0] == 'I' else PipeType.OUTPUT) - self.__handler_id = p[1] - self.__id = p[2] - - def __from_tuple(self, t): - self.__type = t[0] - self.__handler_id = t[1] - self.__id = t[2] - - def __str__(self): - return f"{'I' if self.__type == PipeType.INPUT else 'O'}_{self.__handler_id}_{self.__id}" - - def __eq__(self, pn): - if not isinstance(pn, PipeName): - return NotImplemented - return (str(pn) == str(self)) and (pn.root_path == self.__root) - - @property - def pipe_path(self): - return os.path.join(self.__root, str(self)) - - @property - def type(self): - return self.__type - - @property - def handler_id(self): - return self.__handler_id - - @property - def id(self): - return self.__id - - @property - def root_path(self): - return self.__root - diff --git a/hopper/common/pipe_type.py b/hopper/common/pipe_type.py deleted file mode 100644 index dfae38d..0000000 --- a/hopper/common/pipe_type.py +++ /dev/null @@ -1,3 +0,0 @@ -class PipeType: - OUTPUT, RECEIVING = 0, 0 - INPUT, SENDING = 1, 1 \ No newline at end of file diff --git a/hopper/server/handler.c b/hopper/server/handler.c deleted file mode 100644 index babfa26..0000000 --- a/hopper/server/handler.c +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - -#include "handler.h" - -/// A mapping between a handler ID and name -struct HandlerMapping { - short handler; - char *name; -}; - -/// An array of handler mappings -static struct HandlerMapping HANDLER_MAP[] = { - HANDLER_GENERIC, HANDLER_LOG, - -#ifdef UNUSED_HANDLERS - HANDLER_FULL_LOG, HANDLER_COMPLETE_LOG, -#endif - - HANDLER_START_BUTTON, HANDLER_STARTER, HANDLER_HARDWARE, - HANDLER_HARDWARE_RX, HANDLER_HARDWARE_TX, -}; - -/// Maps a handler string to an ID number -short map_handler_to_id(char *handler) { - int n_handlers = sizeof(HANDLER_MAP) / sizeof(struct HandlerMapping); - - for (int i = 0; i < n_handlers; i++) - if (!strcmp(handler, HANDLER_MAP[i].name)) - return HANDLER_MAP[i].handler; - - printf("pipe handler '%s' is not recognised\n", handler); - - return HANDLER_UNKNOWN; -} diff --git a/hopper/server/handler.h b/hopper/server/handler.h deleted file mode 100644 index dac8ae8..0000000 --- a/hopper/server/handler.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef handler_h_INCLUDED -#define handler_h_INCLUDED - -#define HANDLER_UNKNOWN 0 -#define HANDLER_GENERIC {1, "generic"} -#define HANDLER_LOG {2, "log"} - -#ifdef UNUSED_HANDLERS - -#define HANDLER_FULL_LOG {3, "fulllog"} -#define HANDLER_COMPLETE_LOG {4, "complog"} - -#endif - -#define HANDLER_START_BUTTON {5, "start-button"} -#define HANDLER_STARTER {6, "starter"} -#define HANDLER_HARDWARE {7, "hardware"} -#define HANDLER_HARDWARE_RX {8, "hardware-rx"} -#define HANDLER_HARDWARE_TX {9, "hardware-tx"} - -#define MAX_HANDLER_ID 9 - -short map_handler_to_id(char *handler); - -#endif // handler_h_INCLUDED diff --git a/hopper/server/pipe.c b/hopper/server/pipe.c deleted file mode 100644 index 66f75ac..0000000 --- a/hopper/server/pipe.c +++ /dev/null @@ -1,241 +0,0 @@ -#define _GNU_SOURCE - -#include -#include -#include -#include -#include -#include - -#include "handler.h" -#include "pipe.h" - -/// Close all file descriptors in a PipeSet object -/// Threads should be joined first -void close_pipe_set(struct PipeSet *set) { - close(set->fd); - set->fd = -1; -} - -/// Free a PipeSet, file descriptors should be closed first -void free_pipe_set(struct PipeSet **set) { - if (!set) - return; - - struct PipeSet *_set = *set; - free(_set->info->id); - free(_set->info); - free(_set); - - // Set the pointer to NULL so it isn't reused - (*set) = NULL; -} - -/// Generate a PipeInfo object from a file path -struct PipeInfo *get_pipe_info(const char *path) { - struct PipeInfo *info = (struct PipeInfo *)malloc(sizeof(struct PipeInfo)); - if (!info) { - perror("malloc"); - return NULL; - } - - info->path = strdup(path); - - char *filename = basename((char *)path); - - info->name = strdup(filename); - - char *type = strtok(filename, "_"); - if (!type) - goto err_bad_fname; - - switch (*type) { - case 'I': - info->type = PIPE_SRC; - break; - case 'O': - info->type = PIPE_DST; - break; - default: - goto err_bad_fname; - } - - char *handler = strtok(NULL, "_"); - if (!handler) - goto err_bad_fname; - - info->handler = map_handler_to_id(handler); - - char *id = strtok(NULL, "_"); - if (!id) - goto err_bad_fname; - - int len = strlen(id) + 1; - - info->id = (char *)malloc(sizeof(char) * len); - if (!info->id) { - free(info); - perror("malloc"); - return NULL; - } - - strcpy(info->id, id); - - return info; - -err_bad_fname: - printf("Badly formatted filename: '%s'\n", filename); - free(info); - return NULL; -} - -/// Try to reopen a previously closed pipe -int reopen_pipe_set(struct PipeSet *set, struct HopperData *data) { - if (set->status == PIPE_ACTIVE) - return 0; - - if ((set->fd = open(set->info->path, - (set->info->type == PIPE_SRC ? O_RDONLY : O_WRONLY) | - O_NONBLOCK)) < 0) { - if (errno == ENXIO && set->info->type == PIPE_DST) { - pipe_set_status_inactive(set, data); - return 1; - } - - pipe_set_status_inactive(set, data); - perror("open"); - return 1; - } - - pipe_set_status_active(set, data); - - return 0; -} - -/// Open a PipeSet object from a file -struct PipeSet *open_pipe_set(const char *path) { - char *path_copy = strdup(path); - - struct PipeSet *set = (struct PipeSet *)malloc(sizeof(struct PipeSet)); - if (!set) { - perror("malloc"); - return NULL; - } - - struct PipeInfo *info = get_pipe_info(path_copy); - if (!info) - return NULL; - - set->info = info; - set->status = PIPE_INACTIVE; - set->next = NULL; - set->next_output = NULL; - set->fd = -1; - set->rd_ptr = NULL; - - return set; -} - -ssize_t nb_read(int fd, void *buf, ssize_t max) { - if (max == 0) - return 0; - - ssize_t bytes_copied = 0; - - while (bytes_copied < max) { - ssize_t res = read(fd, buf + bytes_copied, max - bytes_copied); - if (res == -1 && (errno == EAGAIN || errno == EINTR)) - return bytes_copied; - else if (res == -1) { - perror("read"); - return -1; - } - else if (res == 0) - break; - - bytes_copied += res; - } - - return bytes_copied; -} - -ssize_t nb_write(int fd, void *buf, ssize_t max) { - if (max == 0) - return 0; - - ssize_t bytes_copied = 0; - - while (bytes_copied < max) { - ssize_t res = write(fd, buf + bytes_copied, max - bytes_copied); - if (res == -1 && (errno == EAGAIN || errno == EINTR)) - return bytes_copied; - else if (res == -1) { - perror("write"); - return -1; - } - - bytes_copied += res; - } - - return bytes_copied; -} - -ssize_t read_fifo(struct HopperData *data, struct PipeSet *src) { - void *high_read_ptr = get_high_read_ptr(data, src->info->handler); - void *low_read_ptr = get_low_read_ptr(data, src->info->handler); - void *wr_ptr = data->buffers[src->info->handler]->wr_ptr; - ssize_t max_read = 0; - - if (!low_read_ptr || !high_read_ptr || wr_ptr >= high_read_ptr) - max_read = data->buffers[src->info->handler]->buf_end - wr_ptr; - else if (wr_ptr < low_read_ptr) - max_read = low_read_ptr - wr_ptr; - - if (max_read == 0) - return 0; - - ssize_t res = nb_read(src->fd, wr_ptr, max_read); - if (res == -1) - return -1; - - wr_ptr += res; - - if (wr_ptr >= data->buffers[src->info->handler]->buf_end) - wr_ptr = data->buffers[src->info->handler]->buf; - - data->buffers[src->info->handler]->last_wr_ptr = data->buffers[src->info->handler]->wr_ptr; - data->buffers[src->info->handler]->wr_ptr = wr_ptr; - - if (res > 0) - printf("%d/%s -> %zd bytes\n", src->info->handler, src->info->id, res); - - return res; -} - -ssize_t write_fifo(struct HopperData *data, struct PipeSet *dst) { - ssize_t max_write = 0; - - if (dst->rd_ptr > data->buffers[dst->info->handler]->wr_ptr) - max_write = data->buffers[dst->info->handler]->buf_end - dst->rd_ptr; - else if (dst->rd_ptr < data->buffers[dst->info->handler]->wr_ptr) - max_write = data->buffers[dst->info->handler]->wr_ptr - dst->rd_ptr; - - if (max_write == 0) - return 0; - - ssize_t res = nb_write(dst->fd, dst->rd_ptr, max_write); - if (res == -1 && errno == EPIPE) - pipe_set_status_inactive(dst, data); - if (res == -1) - return -1; - - dst->rd_ptr += res; - - if (dst->rd_ptr >= data->buffers[dst->info->handler]->buf_end) - dst->rd_ptr = data->buffers[dst->info->handler]->buf; - - if (res > 0) - printf("%d/%s <- %zd bytes\n", dst->info->handler, dst->info->id, res); - - return res; -} diff --git a/hopper/server/pipe.h b/hopper/server/pipe.h deleted file mode 100644 index 8ac5a54..0000000 --- a/hopper/server/pipe.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef pipe_h_INCLUDED -#define pipe_h_INCLUDED - -#include - -#include "server.h" - -#define PIPE_SRC 1 -#define PIPE_DST 2 - -#define PIPE_INACTIVE 0 -#define PIPE_ACTIVE 1 - -/// A structure containing file information about a I/O pipe -struct PipeInfo { - short type; - short handler; - char *id; - char *name; - char *path; -}; - -/// A structure for holding I/O pipe data -struct PipeSet { - void *rd_ptr; - int fd; - int inotify_fd; - short status; - struct PipeInfo *info; - struct PipeSet *next; - struct PipeSet *next_output; -}; - -void close_pipe_set(struct PipeSet *set); -void free_pipe_set(struct PipeSet **set); -struct PipeInfo *get_pipe_info(const char *path); -struct PipeSet *open_pipe_set(const char *path); -int reopen_pipe_set(struct PipeSet *set, struct HopperData *data); -ssize_t read_fifo(struct HopperData *data, struct PipeSet *src); -ssize_t write_fifo(struct HopperData *data, struct PipeSet *dst); - -#endif // pipe_h_INCLUDED diff --git a/hopper/server/server.c b/hopper/server/server.c deleted file mode 100644 index 002323f..0000000 --- a/hopper/server/server.c +++ /dev/null @@ -1,499 +0,0 @@ -#define _GNU_SOURCE - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "handler.h" -#include "pipe.h" -#include "server.h" - -// Data value used for inotify in epoll. PipeSet FDs use their pointers, so set -// this to a value that would never be a valid pointer. NULL is used for other -// things, 0x1 is low and probably won't be used by a PipeSet pointer. -#define INOTIFY_DATA 0x1 - -void *get_high_read_ptr(struct HopperData *data, short handler) { - void *rd_ptr = NULL; - - struct PipeSet *dst = data->outputs[handler]; - while (dst) { - if (dst->rd_ptr > rd_ptr) - rd_ptr = dst->rd_ptr; - - dst = dst->next_output; - } - - return rd_ptr; -} - -void *get_low_read_ptr(struct HopperData *data, short handler) { - void *rd_ptr = (void *)((uintptr_t)-1); // This is the maximum value for a ptr - - struct PipeSet *dst = data->outputs[handler]; - while (dst) { - if (dst->rd_ptr < rd_ptr && dst->rd_ptr) - rd_ptr = dst->rd_ptr; - - dst = dst->next_output; - } - - // Set to NULL if unchanged for consistency - if (rd_ptr == (void *)((uintptr_t)-1)) - rd_ptr = NULL; - - return rd_ptr; -} - -void free_pipe_list(struct PipeSet *head) { - struct PipeSet *set; - while (head) { - set = head->next; - free(head); - head = set; - } -} - -void prepend_pipe_list(struct PipeSet **head, struct PipeSet *set) { - set->next = *head; - *head = set; -} - -struct PipeSet *remove_pipe_by_name(struct PipeSet **pipes, struct PipeInfo *info) { - struct PipeSet *tgt; - - struct PipeSet head; - struct PipeInfo head_info; - head_info.name = "_HEAD_"; - head.info = &head_info; - head.next = *pipes; - - tgt = &head; - - while (tgt->next) { - //printf("checking %s\n", tgt->next->info->name); - if (!strcmp(tgt->next->info->name, info->name)) { - struct PipeSet *to_free = tgt->next; - // printf("removing %s, connecting %s to %s\n", to_free->info->name, tgt->info->name, tgt->next->next->info->name); - tgt->next = tgt->next->next; - - // This is redundant if the pipe is not at the start of the list - *pipes = head.next; - return to_free; - } - - tgt = tgt->next; - } - - return NULL; -} - -struct PipeSet *remove_output_pipe_by_name(struct PipeSet **pipes, struct PipeInfo *info) { - struct PipeSet *tgt; - - struct PipeSet head; - struct PipeInfo head_info; - head_info.name = "_HEAD_"; - head.info = &head_info; - head.next_output = *pipes; - - tgt = &head; - - while (tgt->next_output) { - //printf("checking %s\n", tgt->next_output->info->name); - if (!strcmp(tgt->next_output->info->name, info->name)) { - struct PipeSet *to_free = tgt->next_output; - // printf("removing %s, connecting %s to %s\n", to_free->info->name, tgt->info->name, tgt->next_output->next_output->info->name); - tgt->next_output = tgt->next_output->next_output; - - // This is redundant if the pipe is not at the start of the list - *pipes = head.next_output; - return to_free; - } - - tgt = tgt->next_output; - } - - return NULL; -} - -void free_hopper_buffer(struct HopperBuffer *buffer) { - if (!buffer) - return; - - if (buffer->buf) - free(buffer->buf); - - free(buffer); -} - -struct HopperBuffer *alloc_hopper_buffer() { - struct HopperBuffer *buffer = (struct HopperBuffer *)malloc(sizeof(struct HopperBuffer)); - if (!buffer) { - perror("alloc"); - return NULL; - } - - buffer->buf = (void *)malloc(MAX_BUF_SIZE); - if (!buffer->buf) { - perror("alloc"); - free(buffer); - return NULL; - } - - buffer->wr_ptr = buffer->buf; - buffer->last_wr_ptr = buffer->buf; - buffer->buf_len = MAX_BUF_SIZE; - buffer->buf_end = buffer->buf + buffer->buf_len; - - return buffer; -} - -/// Safely free a HopperData structure -void free_hopper_data(struct HopperData *data) { - if (!data) - return; - - free_pipe_list(data->pipes); - - if (data->outputs) - free(data->outputs); - - for (int i = 0; i < MAX_HANDLER_ID + 1; i++) { - if (data->buffers[i]) - free(data->buffers[i]); - } - - if (data->buffers) - free(data->buffers); -} - -void close_hopper_fds(struct HopperData *data) { - if (!data) - return; - - close(data->epoll_fd); - close(data->inotify_fd); - close(data->devnull); - - struct PipeSet *set = data->pipes; - - do { - close(set->fd); - set = set->next; - } while (set); -} - -/// Allocate a new HopperData structure -struct HopperData *alloc_hopper_data() { - struct HopperData *data = - (struct HopperData *)malloc(sizeof(struct HopperData)); - if (!data) - goto err_alloc; - - data->outputs = - (struct PipeSet **)calloc(MAX_HANDLER_ID + 1, sizeof(struct PipeSet *)); - if (!data->outputs) - goto err_alloc; - - data->buffers = (struct HopperBuffer **)calloc(MAX_HANDLER_ID + 1, sizeof(struct HopperData *)); - if (!data->buffers) - goto err_alloc; - - for (int i = 0; i < MAX_HANDLER_ID + 1; i++) { - data->buffers[i] = alloc_hopper_buffer(); - if (!data->buffers[i]) - goto err_alloc; - } - - return data; - -err_alloc: - perror("alloc"); - free_hopper_data(data); - return NULL; -} - -int epoll_add_src_pipe(struct HopperData *data, struct PipeSet *set) { - struct epoll_event ev = {}; - ev.events = EPOLLIN; - ev.data.ptr = (void *)set; - - int res; - if ((res = epoll_ctl(data->epoll_fd, EPOLL_CTL_ADD, set->fd, &ev)) != 0) - perror("epoll_ctl ADD"); - - return res; -} - -int load_new_pipe(struct HopperData *data, const char *path) { - struct PipeSet *set = open_pipe_set(path); - if (!set) - return 1; - - prepend_pipe_list(&data->pipes, set); - - if (set->info->type == PIPE_DST) { - set->next_output = data->outputs[set->info->handler]; - data->outputs[set->info->handler] = set; - - set->rd_ptr = NULL; - } - - printf("added fifo '%s'\n", path); - - reopen_pipe_set(set, data); - - return 0; -} - -void pipe_set_status_inactive(struct PipeSet *set, struct HopperData *data) { - if (set->status == PIPE_INACTIVE) - return; - - if (set->info->type == PIPE_SRC) - if (epoll_ctl(data->epoll_fd, EPOLL_CTL_DEL, set->fd, NULL) != 0) - perror("epoll_ctl DEL"); - - close(set->fd); - set->fd = -1; - set->status = PIPE_INACTIVE; - printf("%d/%s set to INACTIVE\n", set->info->handler, set->info->id); -} - -void pipe_set_status_active(struct PipeSet *set, struct HopperData *data) { - if (set->status == PIPE_ACTIVE) - return; - - if (set->info->type == PIPE_SRC) - epoll_add_src_pipe(data, set); - - if (!set->rd_ptr) // For freshly created pipes - set->rd_ptr = data->buffers[set->info->handler]->wr_ptr; - else // Pipes transitioning from inactive get last message - set->rd_ptr = data->buffers[set->info->handler]->last_wr_ptr; - - set->status = PIPE_ACTIVE; - - printf("%d/%s set to ACTIVE\n", set->info->handler, set->info->id); -} - -int load_pipes_directory(struct HopperData *data) { - struct dirent **entries; - int n; - - if ((n = scandir(data->pipe_dir, &entries, NULL, alphasort)) < 0) { - perror("scandir"); - return 1; - } - - for (int i = 0; i < n; i++) { - struct dirent *entry = entries[i]; - - if (entry->d_type == DT_FIFO || entry->d_type == DT_UNKNOWN) { - - char path[PATH_MAX]; - snprintf(path, sizeof(path), "%s/%s", data->pipe_dir, - entry->d_name); - - load_new_pipe(data, path); - } - free(entry); - } - - free(entries); - - return 0; -} - -int handle_inotify_event(struct HopperData *data) { - struct inotify_event *ev = (struct inotify_event *)malloc( - sizeof(struct inotify_event) + NAME_MAX + 1); - - if (read(data->inotify_fd, ev, - sizeof(struct inotify_event) + NAME_MAX + 1) < 0) { - perror("read"); - return 1; - } - - if (ev->mask & IN_DELETE_SELF) { - // The pipes directory got deleted, hopper can't continue, exit - // immediately. - printf("pipes directory disappeared, exiting...\n"); - _exit(1); - } - - if (ev->mask & IN_CREATE) { - char path[PATH_MAX]; - snprintf(path, sizeof(path), "%s/%s", data->pipe_dir, ev->name); - - if (load_new_pipe(data, path) < 0) { - free(ev); - return 1; - } - } - - if (ev->mask & IN_DELETE) { - char path[PATH_MAX]; - snprintf(path, sizeof(path), "%s/%s", data->pipe_dir, ev->name); - - struct PipeInfo *info = get_pipe_info(path); - - if (!info) - return 1; - - int removed = 0; - - struct PipeSet *to_free = remove_pipe_by_name(&data->pipes, info); - if (to_free && info->type == PIPE_DST) { - // Output pipes need to be removed from a separate linked list - // as well as the main one - to_free = remove_output_pipe_by_name(&data->outputs[info->handler], info); - if (to_free) { - free_pipe_set(&to_free); - removed = 1; - } - } else if (to_free) { - free_pipe_set(&to_free); - removed = 1; - } - - if (removed) - printf("removed %d/%s\n", info->handler, info->name); - else - printf("pipe %d/%s not found\n", info->handler, info->name); - } - - free(ev); - return 0; -} - -int flush_and_scan_pipes(struct HopperData *data) { - struct PipeSet *set = data->pipes; - - while (set) { - if (set->status == PIPE_INACTIVE) - reopen_pipe_set(set, data); - - if (set->status == PIPE_ACTIVE && set->info->type == PIPE_DST) - write_fifo(data, set); - - set = set->next; - } - - return 0; -} - -int run_epoll_cycle(struct HopperData *data) { - struct epoll_event events[MAX_EVENTS]; - int res; - - int n = epoll_wait(data->epoll_fd, events, MAX_EVENTS, 250); - if (n < 0) { - perror("epoll_wait"); - return n; - } - - for (int i = 0; i < n; i++) { - if (events[i].data.u64 == INOTIFY_DATA) { - handle_inotify_event(data); - continue; - } - - struct PipeSet *set = (struct PipeSet *)events[i].data.ptr; - - if (events[i].events & EPOLLIN) - if ((res = read_fifo(data, set)) < 0) - return res; - - if (events[i].events & EPOLLHUP) - pipe_set_status_inactive(set, data); - } - - flush_and_scan_pipes(data); - - return n; -} - -int main(int argc, char *argv[]) { - if (argc < 2) { - printf("Usage: %s \n", argv[0]); - return 1; - } - - int ret = 0; - - // Writing to a closed FIFO gives us a SIGPIPE, this is internally handled - // so ignore it. - signal(SIGPIPE, SIG_IGN); - - struct HopperData *data = alloc_hopper_data(); - if (!data) { - ret = 1; - goto cleanup; - } - - data->pipe_dir = argv[1]; - - if ((data->devnull = open("/dev/null", O_WRONLY)) < 0) { - perror("open"); - ret = 1; - goto cleanup; - } - - if ((data->epoll_fd = epoll_create1(0)) < 0) { - perror("epoll_create"); - ret = 1; - goto cleanup; - } - - if ((data->inotify_fd = inotify_init()) < 0) { - perror("inotify_init"); - ret = 1; - goto cleanup; - } - - struct epoll_event ev = {}; - ev.events = EPOLLIN; - ev.data.u64 = - INOTIFY_DATA; // Ensure u64 is used here, not u32, which could be shared - // with a pointer due to size differences. e.g. ptr could - // be 0x7fffffff{INOTIFY_DATA}, using u64 prevents this!! - - if (epoll_ctl(data->epoll_fd, EPOLL_CTL_ADD, data->inotify_fd, &ev) != 0) { - perror("epoll_ctl ADD"); - ret = 1; - goto cleanup; - } - - if ((data->inotify_root_watch_fd = - inotify_add_watch(data->inotify_fd, data->pipe_dir, - IN_CREATE | IN_DELETE | IN_DELETE_SELF)) < 0) { - perror("inotify_add_watch"); - ret = 1; - goto cleanup; - } - - if (load_pipes_directory(data) != 0) { - ret = 1; - goto cleanup; - } - - int res = 0; - while (res >= 0) { - res = run_epoll_cycle(data); - if (res < 0 && errno == EINTR) - res = 0; - } - -cleanup: - close_hopper_fds(data); - free_hopper_data(data); - return ret; -} diff --git a/hopper/server/server.h b/hopper/server/server.h deleted file mode 100644 index b892157..0000000 --- a/hopper/server/server.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef server_h_INCLUDED -#define server_h_INCLUDED - -#include - -#define MAX_EVENTS 64 -#define MAX_COPY_SIZE 1024 * 1024 -#define MAX_BUF_SIZE MAX_COPY_SIZE - -struct HopperBuffer { - void *buf; - void *buf_end; - void *wr_ptr; - void *last_wr_ptr; - ssize_t buf_len; -}; - -struct HopperData { - struct PipeSet *pipes; - struct PipeSet **outputs; - struct HopperBuffer **buffers; - int n_pipes; - int epoll_fd; - int inotify_fd; - int inotify_root_watch_fd; - int devnull; - const char *pipe_dir; -}; - -void pipe_set_status_inactive(struct PipeSet *set, struct HopperData *data); -void pipe_set_status_active(struct PipeSet *set, struct HopperData *data); -void *get_high_read_ptr(struct HopperData *data, short handler); -void *get_low_read_ptr(struct HopperData *data, short handler); - -#endif // server_h_INCLUDED diff --git a/hopper/util/read.py b/hopper/util/read.py deleted file mode 100755 index e8d4a80..0000000 --- a/hopper/util/read.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python3 - -import time, sys, os -from hopper.client import * -from hopper.common import * - -root, name = os.path.split(sys.argv[1]) -pn = PipeName(name, root) -c = HopperClient() -c.open_pipe(pn) - -while True: - buf = c.read(pn) - if buf: - print(buf.decode(), end="") - time.sleep(0.5) - diff --git a/hopper/util/write.py b/hopper/util/write.py deleted file mode 100755 index 909cdb8..0000000 --- a/hopper/util/write.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/python3 - -import os, sys -from hopper.client import * -from hopper.common import * - -root, name = os.path.split(sys.argv[1]) -pn = PipeName(name, root) -c = HopperClient() -c.open_pipe(pn) - -while True: - try: - s = input() - b = bytes(s, "utf-8") - c.write(pn, b) - except: - break; \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 8c6f345..0000000 --- a/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -from setuptools import setup - -setup( - name="hopper", - version="0.1", - packages=["hopper"], - - author="Nathan Gill", - author_email="nathan.j.gill@outlook.com", -) \ No newline at end of file From 1c1e04668ba6ecea8c98f6b60ac86c4b1744fa17 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 13:50:00 +0000 Subject: [PATCH 03/46] add initial (new) project structure --- .envrc | 1 + .github/workflows/build_nix.yml | 33 ++++++++++++++++++ .gitignore | 11 +++--- client/lib.cpp | 7 ++++ client/meson.build | 19 ++++++++++ flake.lock | 61 +++++++++++++++++++++++++++++++++ flake.nix | 50 +++++++++++++++++++++++++++ include/hopper/hopper.h | 14 ++++++++ meson.build | 17 +++++++++ nix/package.nix | 13 +++++++ server/main.cpp | 6 ++++ server/meson.build | 7 ++++ 12 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 .envrc create mode 100644 .github/workflows/build_nix.yml create mode 100644 client/lib.cpp create mode 100644 client/meson.build create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 include/hopper/hopper.h create mode 100644 meson.build create mode 100644 nix/package.nix create mode 100644 server/main.cpp create mode 100644 server/meson.build diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.github/workflows/build_nix.yml b/.github/workflows/build_nix.yml new file mode 100644 index 0000000..db55b63 --- /dev/null +++ b/.github/workflows/build_nix.yml @@ -0,0 +1,33 @@ +name: CI + +on: + push: + branches: [ "main", "hopperx" ] + pull_request: + branches: [ "main", "hopperx" ] + +jobs: + build-x86_64: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v26 + + - name: Build Hopper x86_64 + run: nix build .#hopper.cross.x86_64 + + build-aarch64: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v26 + + - name: Build Hopper aarch64 + run: nix build .#hopper.cross.aarch64 + diff --git a/.gitignore b/.gitignore index e98c1f0..5f20868 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ -pipes/ -__pycache__/ -dist/ +.cache/ +.direnv/ build/ -hopper.egg-info/ -tmplog.txt -.gdb_history -.vscode/ +compile_commands.json +result diff --git a/client/lib.cpp b/client/lib.cpp new file mode 100644 index 0000000..25c70da --- /dev/null +++ b/client/lib.cpp @@ -0,0 +1,7 @@ +#include + +#include "hopper/hopper.h" + +extern "C" { +void hello() { std::cout << "hello world\n" << std::endl; } +} diff --git a/client/meson.build b/client/meson.build new file mode 100644 index 0000000..0d086ed --- /dev/null +++ b/client/meson.build @@ -0,0 +1,19 @@ +client_inc = include_directories('.', '../include') + +libhopper = library( + 'hopper', + 'lib.cpp', + include_directories: client_inc, + version: meson.project_version(), + install: true, +) + +pkg = import('pkgconfig') + +pkg.generate( + libhopper, + name: 'hopper', + description: 'Hopper client library', + subdirs: 'hopper', +) + diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..8a3011e --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1768032153, + "narHash": "sha256-6kD1MdY9fsE6FgSwdnx29hdH2UcBKs3/+JJleMShuJg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3146c6aa9995e7351a398e17470e15305e6e18ff", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "utils": "utils" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..9a44c1b --- /dev/null +++ b/flake.nix @@ -0,0 +1,50 @@ +{ + description = ""; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + utils.url = "github:numtide/flake-utils"; + }; + + outputs = + { + self, + nixpkgs, + utils, + }: + let + systems = [ + "x86_64-linux" + "aarch64-linux" + ]; + in + utils.lib.eachSystem systems ( + system: + let + pkgs = import nixpkgs { inherit system; }; + in + { + packages = { + hopper = { + default = pkgs.callPackage ./nix/package.nix { }; + + cross = { + x86_64 = pkgs.pkgsCross.gnu64.callPackage ./nix/package.nix { }; + x86_64-static = pkgs.pkgsCross.gnu64.pkgsStatic.callPackage ./nix/package.nix { }; + aarch64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage ./nix/package.nix { }; + aarch64-static = pkgs.pkgsCross.aarch64-multiplatform.pkgsStatic.callPackage ./nix/package.nix { }; + }; + }; + }; + + devShell = pkgs.mkShell { + packages = with pkgs; [ + clang-tools + meson + ninja + pkg-config + ]; + }; + } + ); +} diff --git a/include/hopper/hopper.h b/include/hopper/hopper.h new file mode 100644 index 0000000..12bbf2f --- /dev/null +++ b/include/hopper/hopper.h @@ -0,0 +1,14 @@ +#ifndef hopper_h_INCLUDED +#define hopper_h_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +void hello(); + +#ifdef __cplusplus +} +#endif + +#endif // hopper_h_INCLUDED diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..a533bc2 --- /dev/null +++ b/meson.build @@ -0,0 +1,17 @@ +project( + 'hopper', + ['cpp'], + version: '0.1.0', + default_options: ['warning_level=2'], +) + +inc = include_directories('include') + +install_headers( + 'include/hopper/hopper.h', + subdir: 'hopper', +) + +subdir('client') +subdir('server') + diff --git a/nix/package.nix b/nix/package.nix new file mode 100644 index 0000000..3813b80 --- /dev/null +++ b/nix/package.nix @@ -0,0 +1,13 @@ +{ stdenv, meson, ninja, pkg-config }: + +stdenv.mkDerivation { + name = "hopper"; + + src = ./..; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; +} diff --git a/server/main.cpp b/server/main.cpp new file mode 100644 index 0000000..dbba54d --- /dev/null +++ b/server/main.cpp @@ -0,0 +1,6 @@ +#include + +int main(int argc, char *argv[]) { + std::cout << "hello world" << std::endl; + return 0; +} diff --git a/server/meson.build b/server/meson.build new file mode 100644 index 0000000..99413f7 --- /dev/null +++ b/server/meson.build @@ -0,0 +1,7 @@ +executable( + 'hopper', + 'main.cpp', + include_directories: include_directories('.', '../include'), + install: true, +) + From 63eddd64c5e410998888f949a51f96b1e33f6d7f Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 15:56:43 +0000 Subject: [PATCH 04/46] add initial `HopperBuffer` implementation --- include/hopper/server/buffer.hpp | 54 +++++++++++++++++++ include/hopper/server/handler.hpp | 14 +++++ include/hopper/server/hopper.hpp | 22 ++++++++ include/hopper/server/pipe.hpp | 10 ++++ meson.build | 2 +- server/buffer.cpp | 90 +++++++++++++++++++++++++++++++ server/meson.build | 1 + 7 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 include/hopper/server/buffer.hpp create mode 100644 include/hopper/server/handler.hpp create mode 100644 include/hopper/server/hopper.hpp create mode 100644 include/hopper/server/pipe.hpp create mode 100644 server/buffer.cpp diff --git a/include/hopper/server/buffer.hpp b/include/hopper/server/buffer.hpp new file mode 100644 index 0000000..abbe1fb --- /dev/null +++ b/include/hopper/server/buffer.hpp @@ -0,0 +1,54 @@ +#ifndef buffer_hpp_INCLUDED +#define buffer_hpp_INCLUDED + +#include "hopper/server/pipe.hpp" +#include + +namespace hopper { + +enum SeekDirection { + FORWARD, + REVERSE, +}; + +class BufferMarker { +private: + size_t m_pos; + +public: + BufferMarker(size_t pos = 0) : m_pos(pos) {} + + void seek(size_t offset, size_t max, + SeekDirection dir = SeekDirection::FORWARD); + + size_t pos() { return m_pos; } +}; + +class HopperBuffer { +private: + std::vector m_buf; + std::vector m_markers; + + size_t m_edge; + +public: + HopperBuffer(size_t len = 1024 * 1024) + : m_buf(len) {} // Use 1 MiB size by default + + BufferMarker* create_marker(); + + size_t write(void *src, size_t len); + size_t write(const HopperPipe &pipe, size_t len); + + size_t read(BufferMarker *marker, void *dst, size_t len); + size_t read(BufferMarker *marker, const HopperPipe &pipe, size_t len); + + size_t max_write(); + size_t max_read(BufferMarker *marker); + + size_t edge() { return m_edge; } +}; + +}; // namespace hopper + +#endif // buffer_hpp_INCLUDED diff --git a/include/hopper/server/handler.hpp b/include/hopper/server/handler.hpp new file mode 100644 index 0000000..730b5a7 --- /dev/null +++ b/include/hopper/server/handler.hpp @@ -0,0 +1,14 @@ +#ifndef handler_hpp_INCLUDED +#define handler_hpp_INCLUDED + +namespace hopper { + +enum HandlerType { + UNKNOWN, + LOG, + // ... +}; + +}; // namespace hopper + +#endif // handler_hpp_INCLUDED diff --git a/include/hopper/server/hopper.hpp b/include/hopper/server/hopper.hpp new file mode 100644 index 0000000..a17a06d --- /dev/null +++ b/include/hopper/server/hopper.hpp @@ -0,0 +1,22 @@ +#ifndef hopper_hpp_INCLUDED +#define hopper_hpp_INCLUDED + +#include +#include + +#include "hopper/server/buffer.hpp" +#include "hopper/server/handler.hpp" +#include "hopper/server/pipe.hpp" + +namespace hopper { + +class HopperServer { +private: + std::vector m_inputs; + std::map m_buffers; + std::map> m_outputs; +}; + +}; // namespace hopper + +#endif // hopper_hpp_INCLUDED diff --git a/include/hopper/server/pipe.hpp b/include/hopper/server/pipe.hpp new file mode 100644 index 0000000..8dd59cb --- /dev/null +++ b/include/hopper/server/pipe.hpp @@ -0,0 +1,10 @@ +#ifndef pipe_hpp_INCLUDED +#define pipe_hpp_INCLUDED + +namespace hopper { + +class HopperPipe {}; + +}; // namespace hopper + +#endif // pipe_hpp_INCLUDED diff --git a/meson.build b/meson.build index a533bc2..f3a6d57 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'hopper', ['cpp'], version: '0.1.0', - default_options: ['warning_level=2'], + default_options: ['optimization=2', 'warning_level=2'], ) inc = include_directories('include') diff --git a/server/buffer.cpp b/server/buffer.cpp new file mode 100644 index 0000000..fb0f667 --- /dev/null +++ b/server/buffer.cpp @@ -0,0 +1,90 @@ +#include + +#include "hopper/server/buffer.hpp" + +namespace hopper { + +/* BufferMarker */ + +void BufferMarker::seek(size_t offset, size_t max, SeekDirection dir) { + if (dir == SeekDirection::FORWARD) + m_pos = (m_pos + offset) % max; + else + m_pos = ((m_pos > offset) ? m_pos - offset : max - (offset - m_pos)); +} + +/* HopperBuffer */ + +BufferMarker *HopperBuffer::create_marker() { + auto *m = new BufferMarker(0); + m_markers.push_back(m); + return m; +} + +size_t HopperBuffer::write(void *src, size_t len) { + size_t max_len = std::min(len, max_write()); + size_t done_len = 0; + + // Write up to the buffer length, but only if len > buf_len + size_t next_len = std::min((m_buf.size() - m_edge), max_len); + std::memcpy(src, &m_buf[m_edge], next_len); + m_edge = (m_edge + next_len) % m_buf.size(); + + // Enough bytes have been written, return + done_len += next_len; + if (done_len >= max_len) + return done_len; + + // We are guaranteed tp have space for whatever's left + next_len = max_len - next_len; + std::memcpy(reinterpret_cast(src) + done_len, &m_buf[m_edge], + next_len); + m_edge = (m_edge + next_len) % m_buf.size(); + + done_len += next_len; + return done_len; +} + +size_t HopperBuffer::read(BufferMarker *m, void *dst, size_t len) { + // Just see the arithmetic in `write`, it's the same here. + + size_t max_len = std::min(len, max_read(m)); + size_t done_len = 0; + + size_t next_len = std::min((m_buf.size() - m->pos()), max_len); + std::memcpy(&m_buf[m->pos()], dst, next_len); + m->seek(next_len, m_buf.size(), SeekDirection::FORWARD); + + done_len += next_len; + if (done_len >= max_len) + return done_len; + + next_len = max_len - next_len; + std::memcpy(&m_buf[m->pos()], reinterpret_cast(dst) + done_len, + next_len); + m->seek(next_len, m_buf.size(), SeekDirection::FORWARD); + + done_len += next_len; + return done_len; +} + +size_t HopperBuffer::max_write() { + if (m_markers.empty()) + return m_buf.size(); + + size_t min_pos = m_edge; + + for (auto *m : m_markers) + if (m->pos() < min_pos) + min_pos = m->pos(); + + return ((m_edge <= min_pos) ? min_pos - m_edge + : (m_buf.size() - m_edge) + min_pos); +} + +size_t HopperBuffer::max_read(BufferMarker *m) { + return ((m->pos() <= m_edge) ? m_edge - m->pos() + : (m_buf.size() - m->pos()) + m_edge); +} + +}; // namespace hopper diff --git a/server/meson.build b/server/meson.build index 99413f7..0d89a87 100644 --- a/server/meson.build +++ b/server/meson.build @@ -1,6 +1,7 @@ executable( 'hopper', 'main.cpp', + 'buffer.cpp', include_directories: include_directories('.', '../include'), install: true, ) From aab76780d693f9d853a8a2b99fa40f24a310a73a Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 16:05:50 +0000 Subject: [PATCH 05/46] remove `HandlerType` enum, assign IDs dynamically --- include/hopper/server/handler.hpp | 14 -------------- include/hopper/server/hopper.hpp | 8 +++++--- 2 files changed, 5 insertions(+), 17 deletions(-) delete mode 100644 include/hopper/server/handler.hpp diff --git a/include/hopper/server/handler.hpp b/include/hopper/server/handler.hpp deleted file mode 100644 index 730b5a7..0000000 --- a/include/hopper/server/handler.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef handler_hpp_INCLUDED -#define handler_hpp_INCLUDED - -namespace hopper { - -enum HandlerType { - UNKNOWN, - LOG, - // ... -}; - -}; // namespace hopper - -#endif // handler_hpp_INCLUDED diff --git a/include/hopper/server/hopper.hpp b/include/hopper/server/hopper.hpp index a17a06d..93b6979 100644 --- a/include/hopper/server/hopper.hpp +++ b/include/hopper/server/hopper.hpp @@ -2,19 +2,21 @@ #define hopper_hpp_INCLUDED #include +#include #include #include "hopper/server/buffer.hpp" -#include "hopper/server/handler.hpp" #include "hopper/server/pipe.hpp" namespace hopper { class HopperServer { private: + std::map m_handlers; + std::vector m_inputs; - std::map m_buffers; - std::map> m_outputs; + std::map m_buffers; + std::map> m_outputs; }; }; // namespace hopper From e9cc885ed5f3e50147da91ecb6c1757b38b44d68 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 16:07:17 +0000 Subject: [PATCH 06/46] move `hopper.hpp` to `server.hpp` --- include/hopper/server/{hopper.hpp => server.hpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename include/hopper/server/{hopper.hpp => server.hpp} (100%) diff --git a/include/hopper/server/hopper.hpp b/include/hopper/server/server.hpp similarity index 100% rename from include/hopper/server/hopper.hpp rename to include/hopper/server/server.hpp From 481da9b10593508a705d041d50f049fe31f9c2ec Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 16:25:55 +0000 Subject: [PATCH 07/46] add definition for `HopperPipe` class --- include/hopper/server/buffer.hpp | 22 ++++-------------- include/hopper/server/marker.hpp | 28 +++++++++++++++++++++++ include/hopper/server/pipe.hpp | 39 +++++++++++++++++++++++++++++++- server/buffer.cpp | 2 +- 4 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 include/hopper/server/marker.hpp diff --git a/include/hopper/server/buffer.hpp b/include/hopper/server/buffer.hpp index abbe1fb..516857c 100644 --- a/include/hopper/server/buffer.hpp +++ b/include/hopper/server/buffer.hpp @@ -1,28 +1,14 @@ #ifndef buffer_hpp_INCLUDED #define buffer_hpp_INCLUDED -#include "hopper/server/pipe.hpp" #include -namespace hopper { - -enum SeekDirection { - FORWARD, - REVERSE, -}; - -class BufferMarker { -private: - size_t m_pos; +#include "hopper/server/marker.hpp" -public: - BufferMarker(size_t pos = 0) : m_pos(pos) {} - - void seek(size_t offset, size_t max, - SeekDirection dir = SeekDirection::FORWARD); +namespace hopper { - size_t pos() { return m_pos; } -}; +// Opaque class, see hopper/server/pipe.hpp +class HopperPipe; class HopperBuffer { private: diff --git a/include/hopper/server/marker.hpp b/include/hopper/server/marker.hpp new file mode 100644 index 0000000..5cfd0cf --- /dev/null +++ b/include/hopper/server/marker.hpp @@ -0,0 +1,28 @@ +#ifndef marker_hpp_INCLUDED +#define marker_hpp_INCLUDED + +#include + +namespace hopper { + +enum SeekDirection { + FORWARD, + REVERSE, +}; + +class BufferMarker { +private: + size_t m_pos; + +public: + BufferMarker(size_t pos = 0) : m_pos(pos) {} + + void seek(size_t offset, size_t max, + SeekDirection dir = SeekDirection::FORWARD); + + size_t pos() { return m_pos; } +}; + +}; // namespace hopper + +#endif // marker_hpp_INCLUDED diff --git a/include/hopper/server/pipe.hpp b/include/hopper/server/pipe.hpp index 8dd59cb..58d012e 100644 --- a/include/hopper/server/pipe.hpp +++ b/include/hopper/server/pipe.hpp @@ -1,9 +1,46 @@ #ifndef pipe_hpp_INCLUDED #define pipe_hpp_INCLUDED +#include +#include + +#include "hopper/server/marker.hpp" + namespace hopper { -class HopperPipe {}; +enum PipeType { + IN, + OUT, +}; + +class HopperPipe { +private: + BufferMarker *m_marker; + + std::string m_name; + int m_handler; + PipeType m_type; + + std::filesystem::path m_path; + + int m_fd; + +public: + HopperPipe(std::filesystem::path path, BufferMarker *marker); + + size_t write(void *src, size_t len); + size_t read(void *dst, size_t len); + + int fd() { return m_fd; } + + const std::string &name() { return m_name; } + int handler() { return m_handler; } + PipeType type() { return m_type; } + + const std::filesystem::path &path() { return m_path; } + + BufferMarker *marker() { return m_marker; } +}; }; // namespace hopper diff --git a/server/buffer.cpp b/server/buffer.cpp index fb0f667..6810925 100644 --- a/server/buffer.cpp +++ b/server/buffer.cpp @@ -16,7 +16,7 @@ void BufferMarker::seek(size_t offset, size_t max, SeekDirection dir) { /* HopperBuffer */ BufferMarker *HopperBuffer::create_marker() { - auto *m = new BufferMarker(0); + auto *m = new BufferMarker(m_edge); m_markers.push_back(m); return m; } From 174b99f208a950d83aa416bf2aea3fcaecb74bac Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 17:38:10 +0000 Subject: [PATCH 08/46] add initial `HopperPipe` implementation from old `pipe.c` --- include/hopper/server/pipe.hpp | 23 +++++++-- server/meson.build | 1 + server/pipe.cpp | 93 ++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 server/pipe.cpp diff --git a/include/hopper/server/pipe.hpp b/include/hopper/server/pipe.hpp index 58d012e..2cf5492 100644 --- a/include/hopper/server/pipe.hpp +++ b/include/hopper/server/pipe.hpp @@ -13,9 +13,14 @@ enum PipeType { OUT, }; +enum PipeStatus { + ACTIVE, + INACTIVE, +}; + class HopperPipe { private: - BufferMarker *m_marker; + BufferMarker *m_marker = nullptr; std::string m_name; int m_handler; @@ -23,13 +28,19 @@ class HopperPipe { std::filesystem::path m_path; - int m_fd; + int m_fd = -1; + + PipeStatus m_status = PipeStatus::INACTIVE; public: - HopperPipe(std::filesystem::path path, BufferMarker *marker); + HopperPipe(std::string name, int handler, PipeType type, + std::filesystem::path path, BufferMarker *marker = nullptr); - size_t write(void *src, size_t len); - size_t read(void *dst, size_t len); + ~HopperPipe(); + + int open_pipe(); + size_t write_pipe(void *src, size_t len); + size_t read_pipe(void *dst, size_t len); int fd() { return m_fd; } @@ -40,6 +51,8 @@ class HopperPipe { const std::filesystem::path &path() { return m_path; } BufferMarker *marker() { return m_marker; } + + PipeStatus status() { return m_status; } }; }; // namespace hopper diff --git a/server/meson.build b/server/meson.build index 0d89a87..8b1e47a 100644 --- a/server/meson.build +++ b/server/meson.build @@ -2,6 +2,7 @@ executable( 'hopper', 'main.cpp', 'buffer.cpp', + 'pipe.cpp', include_directories: include_directories('.', '../include'), install: true, ) diff --git a/server/pipe.cpp b/server/pipe.cpp new file mode 100644 index 0000000..50bc503 --- /dev/null +++ b/server/pipe.cpp @@ -0,0 +1,93 @@ +#include +#include + +#include "hopper/server/pipe.hpp" + +namespace hopper { + +/* HopperPipe */ + +HopperPipe::HopperPipe(std::string name, int handler, PipeType type, + std::filesystem::path path, BufferMarker *marker) + : m_marker(marker), m_name(name), m_handler(handler), m_type(type), + m_path(path) { + if (this->open_pipe()) + m_status = PipeStatus::ACTIVE; + else + m_status = PipeStatus::INACTIVE; +} + +HopperPipe::~HopperPipe() { + if (m_fd != -1) + close(m_fd); +} + +int HopperPipe::open_pipe() { + if (m_status == PipeStatus::ACTIVE) + return 1; + + int fd = open(m_path.c_str(), + (m_type == PipeType::IN ? O_RDONLY : O_WRONLY) | O_NONBLOCK); + if (fd < 0) { + perror("open"); + return 0; + } + + m_fd = fd; + + return 1; +} + +size_t HopperPipe::write_pipe(void *src, size_t len) { + if (m_type == PipeType::IN) + return -1; + + if (len == 0) + return 0; + + size_t done_len = 0; + + while (done_len < len) { + ssize_t res = write(m_fd, reinterpret_cast(src) + done_len, + len - done_len); + + if (res == -1 && (errno == EAGAIN || errno == EINTR)) + return done_len; + else if (res == -1) { + perror("write"); + return -1; + } + + done_len += res; + } + + return done_len; +} + +size_t HopperPipe::read_pipe(void *dst, size_t len) { + if (m_type == PipeType::OUT) + return -1; + + if (len == 0) + return 0; + + size_t done_len = 0; + + while (done_len < len) { + ssize_t res = read(m_fd, reinterpret_cast(dst) + done_len, + len - done_len); + + if (res == -1 && (errno == EAGAIN || errno == EINTR)) + return done_len; + else if (res == -1) { + perror("read"); + return -1; + } else if (res == 0) + break; + + done_len += res; + } + + return done_len; +} +}; // namespace hopper From 36da3bae55dc62a74cc0307c5dc20390f68e18ee Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 18:07:03 +0000 Subject: [PATCH 09/46] use "daemon" instead of "server" --- {server => daemon}/buffer.cpp | 2 +- {server => daemon}/main.cpp | 0 {server => daemon}/meson.build | 2 +- {server => daemon}/pipe.cpp | 2 +- include/hopper/{server => daemon}/buffer.hpp | 2 +- .../hopper/{server/server.hpp => daemon/daemon.hpp} | 12 ++++++------ include/hopper/{server => daemon}/marker.hpp | 0 include/hopper/{server => daemon}/pipe.hpp | 2 +- meson.build | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) rename {server => daemon}/buffer.cpp (98%) rename {server => daemon}/main.cpp (100%) rename {server => daemon}/meson.build (90%) rename {server => daemon}/pipe.cpp (98%) rename include/hopper/{server => daemon}/buffer.hpp (95%) rename include/hopper/{server/server.hpp => daemon/daemon.hpp} (62%) rename include/hopper/{server => daemon}/marker.hpp (100%) rename include/hopper/{server => daemon}/pipe.hpp (96%) diff --git a/server/buffer.cpp b/daemon/buffer.cpp similarity index 98% rename from server/buffer.cpp rename to daemon/buffer.cpp index 6810925..919b3c0 100644 --- a/server/buffer.cpp +++ b/daemon/buffer.cpp @@ -1,6 +1,6 @@ #include -#include "hopper/server/buffer.hpp" +#include "hopper/daemon/buffer.hpp" namespace hopper { diff --git a/server/main.cpp b/daemon/main.cpp similarity index 100% rename from server/main.cpp rename to daemon/main.cpp diff --git a/server/meson.build b/daemon/meson.build similarity index 90% rename from server/meson.build rename to daemon/meson.build index 8b1e47a..b13d109 100644 --- a/server/meson.build +++ b/daemon/meson.build @@ -1,5 +1,5 @@ executable( - 'hopper', + 'hopperd', 'main.cpp', 'buffer.cpp', 'pipe.cpp', diff --git a/server/pipe.cpp b/daemon/pipe.cpp similarity index 98% rename from server/pipe.cpp rename to daemon/pipe.cpp index 50bc503..c334fc4 100644 --- a/server/pipe.cpp +++ b/daemon/pipe.cpp @@ -1,7 +1,7 @@ #include #include -#include "hopper/server/pipe.hpp" +#include "hopper/daemon/pipe.hpp" namespace hopper { diff --git a/include/hopper/server/buffer.hpp b/include/hopper/daemon/buffer.hpp similarity index 95% rename from include/hopper/server/buffer.hpp rename to include/hopper/daemon/buffer.hpp index 516857c..b6e8624 100644 --- a/include/hopper/server/buffer.hpp +++ b/include/hopper/daemon/buffer.hpp @@ -3,7 +3,7 @@ #include -#include "hopper/server/marker.hpp" +#include "hopper/daemon/marker.hpp" namespace hopper { diff --git a/include/hopper/server/server.hpp b/include/hopper/daemon/daemon.hpp similarity index 62% rename from include/hopper/server/server.hpp rename to include/hopper/daemon/daemon.hpp index 93b6979..604e199 100644 --- a/include/hopper/server/server.hpp +++ b/include/hopper/daemon/daemon.hpp @@ -1,16 +1,16 @@ -#ifndef hopper_hpp_INCLUDED -#define hopper_hpp_INCLUDED +#ifndef daemon_hpp_INCLUDED +#define daemon_hpp_INCLUDED #include #include #include -#include "hopper/server/buffer.hpp" -#include "hopper/server/pipe.hpp" +#include "hopper/daemon/buffer.hpp" +#include "hopper/daemon/pipe.hpp" namespace hopper { -class HopperServer { +class HopperDaemon { private: std::map m_handlers; @@ -21,4 +21,4 @@ class HopperServer { }; // namespace hopper -#endif // hopper_hpp_INCLUDED +#endif // server_hpp_INCLUDED diff --git a/include/hopper/server/marker.hpp b/include/hopper/daemon/marker.hpp similarity index 100% rename from include/hopper/server/marker.hpp rename to include/hopper/daemon/marker.hpp diff --git a/include/hopper/server/pipe.hpp b/include/hopper/daemon/pipe.hpp similarity index 96% rename from include/hopper/server/pipe.hpp rename to include/hopper/daemon/pipe.hpp index 2cf5492..b72352b 100644 --- a/include/hopper/server/pipe.hpp +++ b/include/hopper/daemon/pipe.hpp @@ -4,7 +4,7 @@ #include #include -#include "hopper/server/marker.hpp" +#include "hopper/daemon/marker.hpp" namespace hopper { diff --git a/meson.build b/meson.build index f3a6d57..364f5eb 100644 --- a/meson.build +++ b/meson.build @@ -13,5 +13,5 @@ install_headers( ) subdir('client') -subdir('server') +subdir('daemon') From abc159162301e8bca0077926524db394a14699e0 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 20:31:27 +0000 Subject: [PATCH 10/46] add initial event processing implementation --- daemon/daemon.cpp | 143 +++++++++++++++++++++++++++++ daemon/endpoint.cpp | 13 +++ daemon/main.cpp | 32 ++++++- daemon/meson.build | 2 + include/hopper/daemon/daemon.hpp | 43 +++++++-- include/hopper/daemon/endpoint.hpp | 41 +++++++++ include/hopper/daemon/event.hpp | 23 +++++ include/hopper/daemon/util.hpp | 20 ++++ 8 files changed, 308 insertions(+), 9 deletions(-) create mode 100644 daemon/daemon.cpp create mode 100644 daemon/endpoint.cpp create mode 100644 include/hopper/daemon/endpoint.hpp create mode 100644 include/hopper/daemon/event.hpp create mode 100644 include/hopper/daemon/util.hpp diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp new file mode 100644 index 0000000..e4a00b8 --- /dev/null +++ b/daemon/daemon.cpp @@ -0,0 +1,143 @@ +#include +#include +#include +#include + +#include +#include + +#include "hopper/daemon/daemon.hpp" +#include "hopper/daemon/util.hpp" + +namespace hopper { + +HopperDaemon::HopperDaemon(std::filesystem::path path, int max_events, + int timeout) + : m_path(path), m_max_events(max_events), m_timeout(timeout) { + if (!std::filesystem::exists(path)) { + std::filesystem::create_directories(path); + } + + // just ignore SIGPIPEs + std::signal(SIGPIPE, SIG_IGN); + + if ((m_epoll_fd = epoll_create1(0)) < 0) + throw_errno("epoll_create1"); + + if ((m_inotify_fd = inotify_init()) < 0) + throw_errno("inotify_init"); + + // Set up an event for inotify + struct HopperEvent *inotify_ev = new HopperEvent; + inotify_ev->fd = m_inotify_fd; + inotify_ev->callback = nullptr; + inotify_ev->data.u64 = 0; + + if (add_event(inotify_ev) != 0) + throw_errno("HopperDaemon::add_event"); + + if ((m_inotify_watch_fd = + inotify_add_watch(m_inotify_fd, path.c_str(), + IN_CREATE | IN_DELETE | IN_DELETE_SELF)) < 0) + throw_errno("inotify_add_watch"); +} + +int HopperDaemon::add_event(HopperEvent *event, int events) { + uint64_t event_id = next_event_id(); + event->id = event_id; + + struct epoll_event ev = {}; + ev.events = events; + ev.data.u64 = event_id; + + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, event->fd, &ev) != 0) + return -1; + + m_events.push_back(event); + + return 0; +} + +int HopperDaemon::remove_event(uint64_t id) { + for (size_t i = 0; i < m_events.size(); i++) { + if (m_events[i]->id == id) { + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, m_events[i]->fd, + nullptr) != 0) + return -1; + + m_events.erase(m_events.begin() + i); + break; + } + } + + return 0; +} + +int HopperDaemon::remove_event(HopperEvent *event) { + for (size_t i = 0; i < m_events.size(); i++) { + if (m_events[i] == event) { + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, m_events[i]->fd, + nullptr) != 0) + return -1; + + m_events.erase(m_events.begin() + i); + break; + } + } + + return 0; +} + +HopperEvent *HopperDaemon::get_event(uint64_t id) { + for (size_t i = 0; i < m_events.size(); i++) + if (m_events[i]->id == id) + return m_events[i]; + + return nullptr; +} + +int HopperDaemon::run() { + int res = 0; + + std::cout << "Started event loop, max " << m_max_events + << " events, timeout " << m_timeout << " ms" << std::endl; + + while (res == 0) { + struct epoll_event *events = new struct epoll_event[m_max_events]; + + int n = epoll_wait(m_epoll_fd, events, m_max_events, m_timeout); + if (n < 0) { + delete[] events; + throw_errno("epoll_wait"); + return -1; + } + + HopperEvent *ev; + + for (int i = 0; i < n; i++) { + if ((ev = get_event(i)) == nullptr) + continue; + + if (ev->callback != nullptr) + if ((res = ev->callback(ev) != 0)) + break; + + res = 0; + } + + delete[] events; + + for (const auto &[_, endpoint] : m_endpoints) { + // This is absolutely disgusting, but I haven't thought of a better + // way yet + if ((res = endpoint->refresh(this)) != 0) + break; + + res = 0; + } + } + + return res; +} + +}; // namespace hopper diff --git a/daemon/endpoint.cpp b/daemon/endpoint.cpp new file mode 100644 index 0000000..e3ecec6 --- /dev/null +++ b/daemon/endpoint.cpp @@ -0,0 +1,13 @@ +#include "hopper/daemon/endpoint.hpp" + +namespace hopper { + +HopperEndpoint::HopperEndpoint(std::filesystem::path path): m_path(path) { + +} + +int HopperEndpoint::refresh(HopperDaemon *daemon) { + return 1; +} + +}; diff --git a/daemon/main.cpp b/daemon/main.cpp index dbba54d..235cec3 100644 --- a/daemon/main.cpp +++ b/daemon/main.cpp @@ -1,6 +1,34 @@ +#include +#include #include +#include "hopper/daemon/daemon.hpp" + int main(int argc, char *argv[]) { - std::cout << "hello world" << std::endl; - return 0; + try { + if (argc > 2) { + std::cout << "Usage: hopperd " << std::endl; + return 1; + } else if (argc == 2) { + std::cout << "Using Hopper at " << argv[1] << std::endl; + auto daemon = hopper::HopperDaemon(argv[1]); + return daemon.run(); + } else if (char *p = getenv("HOPPER_PATH")) { + std::cout << "Using Hopper at " << p << std::endl; + auto daemon = hopper::HopperDaemon(p); + return daemon.run(); + } else { + std::cerr + << "Could not find Hopper instance, try setting HOPPER_PATH, " + "or passing as argument!" + << std::endl; + return 1; + } + } catch (const std::exception &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return 1; + } catch (...) { + std::cerr << "Unknown exception" << std::endl; + return 1; + } } diff --git a/daemon/meson.build b/daemon/meson.build index b13d109..1b2c352 100644 --- a/daemon/meson.build +++ b/daemon/meson.build @@ -3,6 +3,8 @@ executable( 'main.cpp', 'buffer.cpp', 'pipe.cpp', + 'daemon.cpp', + 'endpoint.cpp', include_directories: include_directories('.', '../include'), install: true, ) diff --git a/include/hopper/daemon/daemon.hpp b/include/hopper/daemon/daemon.hpp index 604e199..87309c2 100644 --- a/include/hopper/daemon/daemon.hpp +++ b/include/hopper/daemon/daemon.hpp @@ -1,24 +1,53 @@ #ifndef daemon_hpp_INCLUDED #define daemon_hpp_INCLUDED +#include + +#include +#include #include #include #include -#include "hopper/daemon/buffer.hpp" -#include "hopper/daemon/pipe.hpp" +#include "hopper/daemon/endpoint.hpp" +#include "hopper/daemon/event.hpp" namespace hopper { +constexpr uint64_t INOTIFY_DATA = 0x1; + class HopperDaemon { private: - std::map m_handlers; + std::map m_endpoint_ids; + std::map m_endpoints; + std::vector m_events; + + std::filesystem::path m_path; + + uint64_t m_last_event_id = 0; + + int m_inotify_fd = -1; + int m_inotify_watch_fd = -1; + + int m_epoll_fd = -1; + + int m_max_events = 64; + int m_timeout = 250; + + uint64_t next_event_id() { return m_last_event_id++; } + +public: + HopperDaemon(std::filesystem::path path, int max_events = 64, + int m_timeout = 250); + + int run(); - std::vector m_inputs; - std::map m_buffers; - std::map> m_outputs; + int add_event(HopperEvent *event, int events = EPOLLIN); + int remove_event(uint64_t id); + int remove_event(HopperEvent *event); + HopperEvent *get_event(uint64_t id); }; }; // namespace hopper -#endif // server_hpp_INCLUDED +#endif // daemon_hpp_INCLUDED diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp new file mode 100644 index 0000000..7aba4dd --- /dev/null +++ b/include/hopper/daemon/endpoint.hpp @@ -0,0 +1,41 @@ +#ifndef endpoint_hpp_INCLUDED +#define endpoint_hpp_INCLUDED + +#include +#include + +#include "hopper/daemon/buffer.hpp" +#include "hopper/daemon/event.hpp" +#include "hopper/daemon/pipe.hpp" + +namespace hopper { + +class HopperDaemon; + +class HopperEndpoint { +private: + std::map m_inputs; + std::map m_outputs; + std::vector m_events; + + HopperBuffer m_buffer; + + std::filesystem::path m_path; + + int m_inotify_watch_in; + int m_inotify_watch_out; + + std::string m_name; + +public: + HopperEndpoint(std::filesystem::path path); + + // This is a weird fucntion. This is called by the daemon once every loop + // iteration, which then allows endpoints to manipulate daemon state such as + // updating events, etc. + int refresh(HopperDaemon *d); +}; + +}; // namespace hopper + +#endif // endpoint_hpp_INCLUDED diff --git a/include/hopper/daemon/event.hpp b/include/hopper/daemon/event.hpp new file mode 100644 index 0000000..81f6ad1 --- /dev/null +++ b/include/hopper/daemon/event.hpp @@ -0,0 +1,23 @@ +#ifndef event_hpp_INCLUDED +#define event_hpp_INCLUDED + +#include +#include + +namespace hopper { + +union HopperEventData { + uint64_t u64; + void *ptr; +}; + +struct HopperEvent { + int fd; + uint64_t id; + HopperEventData data; + std::function callback; +}; + +}; // namespace hopper + +#endif // event_hpp_INCLUDED diff --git a/include/hopper/daemon/util.hpp b/include/hopper/daemon/util.hpp new file mode 100644 index 0000000..a6b535d --- /dev/null +++ b/include/hopper/daemon/util.hpp @@ -0,0 +1,20 @@ +#ifndef util_hpp_INCLUDED +#define util_hpp_INCLUDED + +#include +#include +#include +#include +#include + +namespace hopper { + +inline void throw_errno(const std::string &msg) { + std::stringstream ss; + ss << msg << ": " << std::strerror(errno); + throw std::system_error(errno, std::generic_category(), ss.str()); +} + +}; // namespace hopper + +#endif // util_hpp_INCLUDED From a2354fe16ceb4a6fc7e9093b9993adf583164b17 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 11 Jan 2026 22:37:23 +0000 Subject: [PATCH 11/46] add initialization for `HopperEndpoint` --- daemon/daemon.cpp | 109 ++++++++++++++++++++++++++--- daemon/endpoint.cpp | 83 +++++++++++++++++++++- include/hopper/daemon/daemon.hpp | 11 ++- include/hopper/daemon/endpoint.hpp | 33 ++++++++- 4 files changed, 218 insertions(+), 18 deletions(-) diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp index e4a00b8..519c161 100644 --- a/daemon/daemon.cpp +++ b/daemon/daemon.cpp @@ -1,12 +1,16 @@ #include +#include #include +#include #include #include #include #include +#include #include "hopper/daemon/daemon.hpp" +#include "hopper/daemon/endpoint.hpp" #include "hopper/daemon/util.hpp" namespace hopper { @@ -30,9 +34,13 @@ HopperDaemon::HopperDaemon(std::filesystem::path path, int max_events, // Set up an event for inotify struct HopperEvent *inotify_ev = new HopperEvent; inotify_ev->fd = m_inotify_fd; - inotify_ev->callback = nullptr; inotify_ev->data.u64 = 0; + // C++ lambda syntax is really weird + inotify_ev->callback = [this](HopperEvent *ev) { + return this->handle_inotify(ev); + }; + if (add_event(inotify_ev) != 0) throw_errno("HopperDaemon::add_event"); @@ -42,6 +50,86 @@ HopperDaemon::HopperDaemon(std::filesystem::path path, int max_events, throw_errno("inotify_add_watch"); } +HopperDaemon::~HopperDaemon() { + for (const auto &[_, endpoint] : m_endpoints) + delete endpoint; + + for (auto *event : m_events) + delete event; +} + +int HopperDaemon::create_endpoint(std::filesystem::path path) { + int endpoint_id = next_endpoint_id(); + auto *endpoint = new HopperEndpoint(endpoint_id, path); + m_endpoints[endpoint_id] = endpoint; + + std::cout << "CREATE " << endpoint->path() << std::endl; + + return endpoint->refresh(this); +} + +int HopperDaemon::delete_endpoint(int id) { + if (m_endpoints[id] == nullptr) + return 1; + + std::cout << "DELETE " << m_endpoints[id]->path() << std::endl; + + delete m_endpoints[id]; + m_endpoints.erase(id); + + return 0; +} + +int HopperDaemon::delete_endpoint(std::filesystem::path path) { + for (const auto &[_, endpoint] : m_endpoints) { + if (endpoint->path() == path) { + return delete_endpoint(endpoint->id()); + } + } + return 1; +} + +int HopperDaemon::handle_inotify(HopperEvent *ev) { + // I can't remember how to do this with new + struct inotify_event *iev = reinterpret_cast( + std::malloc(sizeof(struct inotify_event) + NAME_MAX + 1)); + + if (read(ev->fd, iev, sizeof(struct inotify_event) + NAME_MAX + 1) < 0) { + throw_errno("read"); + return -1; + } + + if (iev->mask & IN_DELETE_SELF) { + // The hopper got deleted, this is fatal + std::cerr << "(ENOENT) Hopper " << m_path + << " was deleted, exiting... :("; + _exit(1); + } + + if (iev->mask & IN_CREATE) { + std::string path; + path.resize(PATH_MAX); + std::snprintf(path.data(), PATH_MAX, "%s/%s", m_path.c_str(), + iev->name); + + std::free(iev); + return create_endpoint(path); + } + + if (iev->mask & IN_DELETE) { + std::string path; + path.resize(PATH_MAX); + std::snprintf(path.data(), PATH_MAX, "%s/%s", m_path.c_str(), + iev->name); + + std::free(iev); + return delete_endpoint(path); + } + + std::free(iev); + return 0; +} + int HopperDaemon::add_event(HopperEvent *event, int events) { uint64_t event_id = next_event_id(); event->id = event_id; @@ -118,11 +206,12 @@ int HopperDaemon::run() { if ((ev = get_event(i)) == nullptr) continue; - if (ev->callback != nullptr) - if ((res = ev->callback(ev) != 0)) - break; - - res = 0; + if (ev->callback != nullptr) { + int r = ev->callback(ev); + if (r != 0) + std::cerr << "Failed to run callback for Ev(fd=" << ev->fd + << "), code " << r << std::endl; + } } delete[] events; @@ -130,10 +219,10 @@ int HopperDaemon::run() { for (const auto &[_, endpoint] : m_endpoints) { // This is absolutely disgusting, but I haven't thought of a better // way yet - if ((res = endpoint->refresh(this)) != 0) - break; - - res = 0; + int r = endpoint->refresh(this); + if (r != 0) + std::cerr << "Failed to run refresh for Endpoint(path=" + << endpoint->path() << ")" << std::endl; } } diff --git a/daemon/endpoint.cpp b/daemon/endpoint.cpp index e3ecec6..9be1e80 100644 --- a/daemon/endpoint.cpp +++ b/daemon/endpoint.cpp @@ -1,13 +1,90 @@ +#include +#include +#include + +#include "hopper/daemon/daemon.hpp" #include "hopper/daemon/endpoint.hpp" +#include "hopper/daemon/util.hpp" namespace hopper { -HopperEndpoint::HopperEndpoint(std::filesystem::path path): m_path(path) { +HopperEndpoint::HopperEndpoint(int id, std::filesystem::path path) + : m_path(path), m_id(id) { + m_name = path.filename(); + + // Why have I made my life so hard? + + m_in_dir = path / "in"; + std::pair in_notify = create_inotify(m_in_dir, nullptr); + validate_inotify(in_notify); + m_inotify_in_fd = in_notify.first; + m_inotify_in_watch = in_notify.second; + + m_out_dir = path / "out"; + std::pair out_notify = create_inotify(m_out_dir, nullptr); + validate_inotify(out_notify); + m_inotify_out_fd = out_notify.first; + m_inotify_out_watch = out_notify.second; +} + +std::pair +HopperEndpoint::create_inotify(std::filesystem::path path, + std::function callback) { + if (!std::filesystem::exists(path)) + std::filesystem::create_directories(path); + + int fd = inotify_init(); + if (fd < 0) + return std::make_pair(-1, -1); + + int watch = inotify_add_watch(fd, path.c_str(), IN_CREATE | IN_DELETE); + if (watch < 0) + return std::make_pair(fd, -1); + + HopperEvent *ev = new HopperEvent; + ev->fd = fd; + ev->data.u64 = 0; + ev->callback = callback; + HopperEndpointOperation *add_ev = new HopperEndpointOperation(); + add_ev->ev = ev; + add_ev->type = HopperEndpointOperationType::CREATE_EV; + + m_operations.push_back(add_ev); + + return std::make_pair(fd, watch); +} + +void HopperEndpoint::validate_inotify(std::pair &inotify) { + if (inotify.first == -1) + throw_errno("inotify_init"); + if (inotify.second == -1) + throw_errno("inotify_add_watch"); } int HopperEndpoint::refresh(HopperDaemon *daemon) { - return 1; + while (!m_operations.empty()) { + HopperEndpointOperation *op = m_operations.back(); + + int r = 0; + switch (op->type) { + case HopperEndpointOperationType::CREATE_EV: + r = daemon->add_event(op->ev); + break; + case HopperEndpointOperationType::DELETE_EV: + r = daemon->remove_event(op->ev->id); + break; + } + + delete op; + + m_operations.pop_back(); + + if (r != 0) + return r; + } + + return 0; } -}; +}; // namespace hopper diff --git a/include/hopper/daemon/daemon.hpp b/include/hopper/daemon/daemon.hpp index 87309c2..d67a5d5 100644 --- a/include/hopper/daemon/daemon.hpp +++ b/include/hopper/daemon/daemon.hpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include "hopper/daemon/endpoint.hpp" @@ -18,13 +17,13 @@ constexpr uint64_t INOTIFY_DATA = 0x1; class HopperDaemon { private: - std::map m_endpoint_ids; std::map m_endpoints; std::vector m_events; std::filesystem::path m_path; uint64_t m_last_event_id = 0; + int m_last_endpoint_id = 0; int m_inotify_fd = -1; int m_inotify_watch_fd = -1; @@ -35,10 +34,18 @@ class HopperDaemon { int m_timeout = 250; uint64_t next_event_id() { return m_last_event_id++; } + int next_endpoint_id() { return m_last_endpoint_id++; } + + int create_endpoint(std::filesystem::path path); + int delete_endpoint(std::filesystem::path path); + int delete_endpoint(int id); + + int handle_inotify(HopperEvent *ev); public: HopperDaemon(std::filesystem::path path, int max_events = 64, int m_timeout = 250); + ~HopperDaemon(); int run(); diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp index 7aba4dd..3248c2d 100644 --- a/include/hopper/daemon/endpoint.hpp +++ b/include/hopper/daemon/endpoint.hpp @@ -12,28 +12,55 @@ namespace hopper { class HopperDaemon; +enum HopperEndpointOperationType { + CREATE_EV, + DELETE_EV, +}; + +struct HopperEndpointOperation { + HopperEndpointOperationType type; + HopperEvent *ev; +}; + class HopperEndpoint { private: std::map m_inputs; std::map m_outputs; std::vector m_events; + std::vector m_operations; HopperBuffer m_buffer; std::filesystem::path m_path; + std::filesystem::path m_in_dir; + std::filesystem::path m_out_dir; - int m_inotify_watch_in; - int m_inotify_watch_out; + int m_inotify_in_fd; + int m_inotify_in_watch; + + int m_inotify_out_fd; + int m_inotify_out_watch; std::string m_name; + int m_id; + + // First value is inotify fd, second is watch id + std::pair + create_inotify(std::filesystem::path path, + std::function callback); + void validate_inotify(std::pair &inotify); public: - HopperEndpoint(std::filesystem::path path); + HopperEndpoint(int id, std::filesystem::path path); // This is a weird fucntion. This is called by the daemon once every loop // iteration, which then allows endpoints to manipulate daemon state such as // updating events, etc. int refresh(HopperDaemon *d); + + int id() { return m_id; } + const std::string &name() { return m_name; } + const std::filesystem::path &path() { return m_path; } }; }; // namespace hopper From b7b583db0725c600c074af22e0d1c53a777f927a Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Mon, 12 Jan 2026 22:22:05 +0000 Subject: [PATCH 12/46] restructure to avoid weird event things, also open pipes --- daemon/daemon.cpp | 198 ++++++----------------------- daemon/daemon_endpoint.cpp | 51 ++++++++ daemon/daemon_inotify.cpp | 97 ++++++++++++++ daemon/endpoint.cpp | 124 +++++++++--------- daemon/meson.build | 3 + daemon/pipe.cpp | 29 +++-- daemon/util.cpp | 18 +++ include/hopper/daemon/daemon.hpp | 46 ++++--- include/hopper/daemon/endpoint.hpp | 78 ++++++------ include/hopper/daemon/event.hpp | 23 ---- include/hopper/daemon/pipe.hpp | 22 +--- include/hopper/daemon/util.hpp | 4 + meson.build | 2 +- 13 files changed, 360 insertions(+), 335 deletions(-) create mode 100644 daemon/daemon_endpoint.cpp create mode 100644 daemon/daemon_inotify.cpp create mode 100644 daemon/util.cpp delete mode 100644 include/hopper/daemon/event.hpp diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp index 519c161..dee5182 100644 --- a/daemon/daemon.cpp +++ b/daemon/daemon.cpp @@ -1,9 +1,6 @@ #include -#include #include #include -#include -#include #include #include @@ -11,13 +8,14 @@ #include "hopper/daemon/daemon.hpp" #include "hopper/daemon/endpoint.hpp" +#include "hopper/daemon/pipe.hpp" #include "hopper/daemon/util.hpp" namespace hopper { HopperDaemon::HopperDaemon(std::filesystem::path path, int max_events, int timeout) - : m_path(path), m_max_events(max_events), m_timeout(timeout) { + : m_max_events(max_events), m_timeout(timeout), m_path(path) { if (!std::filesystem::exists(path)) { std::filesystem::create_directories(path); } @@ -28,160 +26,59 @@ HopperDaemon::HopperDaemon(std::filesystem::path path, int max_events, if ((m_epoll_fd = epoll_create1(0)) < 0) throw_errno("epoll_create1"); - if ((m_inotify_fd = inotify_init()) < 0) - throw_errno("inotify_init"); - - // Set up an event for inotify - struct HopperEvent *inotify_ev = new HopperEvent; - inotify_ev->fd = m_inotify_fd; - inotify_ev->data.u64 = 0; - - // C++ lambda syntax is really weird - inotify_ev->callback = [this](HopperEvent *ev) { - return this->handle_inotify(ev); - }; - - if (add_event(inotify_ev) != 0) - throw_errno("HopperDaemon::add_event"); - - if ((m_inotify_watch_fd = - inotify_add_watch(m_inotify_fd, path.c_str(), - IN_CREATE | IN_DELETE | IN_DELETE_SELF)) < 0) - throw_errno("inotify_add_watch"); + setup_inotify(); } HopperDaemon::~HopperDaemon() { for (const auto &[_, endpoint] : m_endpoints) delete endpoint; - - for (auto *event : m_events) - delete event; -} - -int HopperDaemon::create_endpoint(std::filesystem::path path) { - int endpoint_id = next_endpoint_id(); - auto *endpoint = new HopperEndpoint(endpoint_id, path); - m_endpoints[endpoint_id] = endpoint; - - std::cout << "CREATE " << endpoint->path() << std::endl; - - return endpoint->refresh(this); } -int HopperDaemon::delete_endpoint(int id) { - if (m_endpoints[id] == nullptr) - return 1; +void HopperDaemon::try_add_pipe(std::pair pipe, PipeType type) { + // Pipe has bad ID or bad FD + if (pipe.first == 0 || pipe.second == -1) + return; - std::cout << "DELETE " << m_endpoints[id]->path() << std::endl; - - delete m_endpoints[id]; - m_endpoints.erase(id); + struct epoll_event ev = {}; + ev.events = (type == PipeType::IN ? EPOLLIN | EPOLLHUP : EPOLLHUP); + ev.data.u64 = pipe.first; - return 0; + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, pipe.second, &ev) != 0) + throw_errno("epoll_ctl ADD"); } -int HopperDaemon::delete_endpoint(std::filesystem::path path) { +void HopperDaemon::refresh_pipes() { + // Try to open any inactive pipes again for (const auto &[_, endpoint] : m_endpoints) { - if (endpoint->path() == path) { - return delete_endpoint(endpoint->id()); + for (const auto &[id, pipe] : endpoint->outputs()) { + if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) + continue; + try_add_pipe(std::make_pair(id, pipe->fd()), PipeType::OUT); } } - return 1; -} - -int HopperDaemon::handle_inotify(HopperEvent *ev) { - // I can't remember how to do this with new - struct inotify_event *iev = reinterpret_cast( - std::malloc(sizeof(struct inotify_event) + NAME_MAX + 1)); - - if (read(ev->fd, iev, sizeof(struct inotify_event) + NAME_MAX + 1) < 0) { - throw_errno("read"); - return -1; - } - - if (iev->mask & IN_DELETE_SELF) { - // The hopper got deleted, this is fatal - std::cerr << "(ENOENT) Hopper " << m_path - << " was deleted, exiting... :("; - _exit(1); - } - - if (iev->mask & IN_CREATE) { - std::string path; - path.resize(PATH_MAX); - std::snprintf(path.data(), PATH_MAX, "%s/%s", m_path.c_str(), - iev->name); - - std::free(iev); - return create_endpoint(path); - } - - if (iev->mask & IN_DELETE) { - std::string path; - path.resize(PATH_MAX); - std::snprintf(path.data(), PATH_MAX, "%s/%s", m_path.c_str(), - iev->name); - - std::free(iev); - return delete_endpoint(path); - } - - std::free(iev); - return 0; -} - -int HopperDaemon::add_event(HopperEvent *event, int events) { - uint64_t event_id = next_event_id(); - event->id = event_id; - - struct epoll_event ev = {}; - ev.events = events; - ev.data.u64 = event_id; - - if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, event->fd, &ev) != 0) - return -1; - - m_events.push_back(event); - - return 0; } -int HopperDaemon::remove_event(uint64_t id) { - for (size_t i = 0; i < m_events.size(); i++) { - if (m_events[i]->id == id) { - if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, m_events[i]->fd, - nullptr) != 0) - return -1; +void HopperDaemon::process_events(struct epoll_event *events, int n_events) { + for (int i = 0; i < n_events; i++) { + struct epoll_event ev = events[i]; - m_events.erase(m_events.begin() + i); - break; + // inotify events use 0 as ID + if (ev.data.u64 == 0) { + handle_inotify(); + continue; } - } - return 0; -} + uint32_t endpoint_id = (ev.data.u64 >> 40) & 0xFFFFFFFFFF; + if (!m_endpoints.contains(endpoint_id)) + continue; -int HopperDaemon::remove_event(HopperEvent *event) { - for (size_t i = 0; i < m_events.size(); i++) { - if (m_events[i] == event) { - if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, m_events[i]->fd, - nullptr) != 0) - return -1; + HopperEndpoint *endpoint = m_endpoints[endpoint_id]; - m_events.erase(m_events.begin() + i); - break; - } + if (ev.events & EPOLLIN) + endpoint->on_pipe_readable(ev.data.u64); + else if (ev.events & EPOLLOUT) + endpoint->on_pipe_writable(ev.data.u64); } - - return 0; -} - -HopperEvent *HopperDaemon::get_event(uint64_t id) { - for (size_t i = 0; i < m_events.size(); i++) - if (m_events[i]->id == id) - return m_events[i]; - - return nullptr; } int HopperDaemon::run() { @@ -195,35 +92,16 @@ int HopperDaemon::run() { int n = epoll_wait(m_epoll_fd, events, m_max_events, m_timeout); if (n < 0) { + if (errno == EINTR) + continue; + delete[] events; throw_errno("epoll_wait"); return -1; } - HopperEvent *ev; - - for (int i = 0; i < n; i++) { - if ((ev = get_event(i)) == nullptr) - continue; - - if (ev->callback != nullptr) { - int r = ev->callback(ev); - if (r != 0) - std::cerr << "Failed to run callback for Ev(fd=" << ev->fd - << "), code " << r << std::endl; - } - } - - delete[] events; - - for (const auto &[_, endpoint] : m_endpoints) { - // This is absolutely disgusting, but I haven't thought of a better - // way yet - int r = endpoint->refresh(this); - if (r != 0) - std::cerr << "Failed to run refresh for Endpoint(path=" - << endpoint->path() << ")" << std::endl; - } + process_events(events, n); + refresh_pipes(); } return res; diff --git a/daemon/daemon_endpoint.cpp b/daemon/daemon_endpoint.cpp new file mode 100644 index 0000000..daee4f5 --- /dev/null +++ b/daemon/daemon_endpoint.cpp @@ -0,0 +1,51 @@ +#include "hopper/daemon/daemon.hpp" +#include + +namespace hopper { + +uint32_t HopperDaemon::create_endpoint(const std::filesystem::path &path) { + uint32_t endpoint_id = next_endpoint_id(); + if (endpoint_id == 0) + return 0; + + int inotify_watch_fd = + inotify_add_watch(m_inotify_fd, path.c_str(), IN_CREATE | IN_DELETE); + if (inotify_watch_fd < 0) { + return 0; + } + + auto *endpoint = new HopperEndpoint(endpoint_id, inotify_watch_fd, path); + m_endpoints[endpoint_id] = endpoint; + + std::cout << "CREATE " << endpoint->path() << std::endl; + + return endpoint_id; +} + +void HopperDaemon::delete_endpoint(uint32_t id) { + if (!m_endpoints.contains(id)) + return; + + std::cout << "DELETE " << m_endpoints[id]->path() << std::endl; + + delete m_endpoints[id]; + m_endpoints.erase(id); +} + +void HopperDaemon::delete_endpoint(const std::filesystem::path &path) { + for (const auto &[_, endpoint] : m_endpoints) { + if (endpoint->path() == path) { + delete_endpoint(endpoint->id()); + break; + } + } +} + +HopperEndpoint *HopperDaemon::endpoint_by_watch(int watch) { + for (const auto &[_, endpoint] : m_endpoints) + if (endpoint->watch_fd() == watch) + return endpoint; + return nullptr; +} + +}; // namespace hopper diff --git a/daemon/daemon_inotify.cpp b/daemon/daemon_inotify.cpp new file mode 100644 index 0000000..685b64f --- /dev/null +++ b/daemon/daemon_inotify.cpp @@ -0,0 +1,97 @@ +#include "hopper/daemon/daemon.hpp" +#include "hopper/daemon/util.hpp" + +#include +#include +#include + +namespace hopper { + +void HopperDaemon::setup_inotify() { + if ((m_inotify_fd = inotify_init()) < 0) + throw_errno("inotify_init"); + + struct epoll_event inotify_ev = {}; + inotify_ev.events = EPOLLIN; + inotify_ev.data.u64 = 0; + + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, m_inotify_fd, &inotify_ev) != 0) + throw_errno("epoll_ctl ADD"); + + if ((m_inotify_root_watch = inotify_add_watch( + m_inotify_fd, m_path.c_str(), + IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_ISDIR)) < 0) + throw_errno("inotify_add_watch"); +} + +void HopperDaemon::handle_root_inotify(struct inotify_event *ev) { + if (ev->mask & IN_DELETE_SELF) { + std::cerr << "Hopper " << m_path << " got deleted, exiting... :(" + << std::endl; + _exit(1); + } + + if (ev->mask & IN_CREATE) { + std::filesystem::path p = m_path; + p /= ev->name; + + if (create_endpoint(p) == 0) + std::cerr << "Endpoint creation failed! Out of IDs?" << std::endl; + } + + if (ev->mask & IN_DELETE) { + std::filesystem::path p = m_path; + p /= ev->name; + + delete_endpoint(p); + } +} + +void HopperDaemon::handle_endpoint_inotify(struct inotify_event *ev, + HopperEndpoint *endpoint) { + if (ev->mask & IN_CREATE) { + std::filesystem::path p = endpoint->path(); + p /= ev->name; + + PipeType pipe_type = detect_pipe_type(p); + if (pipe_type == PipeType::NONE) + return; + + std::pair pipe = + (pipe_type == PipeType::IN ? endpoint->add_input_pipe(p) + : endpoint->add_output_pipe(p)); + try_add_pipe(pipe, pipe_type); + } +} + +void HopperDaemon::handle_inotify() { + struct inotify_event *iev = reinterpret_cast( + std::malloc(sizeof(struct inotify_event) + NAME_MAX + 1)); + + if (read(m_inotify_fd, iev, sizeof(struct inotify_event) + NAME_MAX + 1) <= + 0) + return; + + if (iev->wd == m_inotify_root_watch) { + handle_root_inotify(iev); + std::free(iev); + return; + } + + HopperEndpoint *endpoint = endpoint_by_watch(iev->wd); + if (endpoint == nullptr) { + std::cout << "No endpoint found for watch ID " << iev->wd << std::endl; + std::free(iev); + return; + } + + handle_endpoint_inotify(iev, endpoint); + + // The endpoint is now closed + if (iev->mask & IN_IGNORED) + delete_endpoint(endpoint->id()); + + std::free(iev); +} + +}; // namespace hopper diff --git a/daemon/endpoint.cpp b/daemon/endpoint.cpp index 9be1e80..9dbb2fe 100644 --- a/daemon/endpoint.cpp +++ b/daemon/endpoint.cpp @@ -1,90 +1,82 @@ #include -#include -#include +#include -#include "hopper/daemon/daemon.hpp" #include "hopper/daemon/endpoint.hpp" -#include "hopper/daemon/util.hpp" +#include "hopper/daemon/pipe.hpp" namespace hopper { -HopperEndpoint::HopperEndpoint(int id, std::filesystem::path path) - : m_path(path), m_id(id) { +HopperEndpoint::HopperEndpoint(uint32_t id, int watch_fd, std::filesystem::path path) + : m_path(path), m_id(id), m_watch_fd(watch_fd) { m_name = path.filename(); - - // Why have I made my life so hard? - - m_in_dir = path / "in"; - std::pair in_notify = create_inotify(m_in_dir, nullptr); - validate_inotify(in_notify); - m_inotify_in_fd = in_notify.first; - m_inotify_in_watch = in_notify.second; - - m_out_dir = path / "out"; - std::pair out_notify = create_inotify(m_out_dir, nullptr); - validate_inotify(out_notify); - m_inotify_out_fd = out_notify.first; - m_inotify_out_watch = out_notify.second; } -std::pair -HopperEndpoint::create_inotify(std::filesystem::path path, - std::function callback) { - if (!std::filesystem::exists(path)) - std::filesystem::create_directories(path); - - int fd = inotify_init(); - if (fd < 0) - return std::make_pair(-1, -1); +HopperEndpoint::~HopperEndpoint() { + for (const auto &[_, pipe] : m_inputs) + delete pipe; + for (const auto &[_, pipe] : m_outputs) + delete pipe; +} - int watch = inotify_add_watch(fd, path.c_str(), IN_CREATE | IN_DELETE); - if (watch < 0) - return std::make_pair(fd, -1); +void HopperEndpoint::on_pipe_readable(uint64_t id) { + std::cout << "ID " << id << " is readable\n"; +} - HopperEvent *ev = new HopperEvent; - ev->fd = fd; - ev->data.u64 = 0; - ev->callback = callback; +void HopperEndpoint::on_pipe_writable(uint64_t id) { + std::cout << "ID " << id << " is writable\n"; +} - HopperEndpointOperation *add_ev = new HopperEndpointOperation(); - add_ev->ev = ev; - add_ev->type = HopperEndpointOperationType::CREATE_EV; +std::pair HopperEndpoint::add_input_pipe(const std::filesystem::path &path) { + if (!std::filesystem::is_fifo(path)) + return std::make_pair(0, -1); + + uint64_t id = next_pipe_id(1); // Type 1 for input + if (id == 0) // ID 0 is never valid + return std::make_pair(0, -1); - m_operations.push_back(add_ev); + std::cout << "OPEN IN " << path << "\n"; + + HopperPipe *p = new HopperPipe(id, PipeType::IN, path, nullptr); + m_inputs[id] = p; - return std::make_pair(fd, watch); + return std::make_pair(id, p->fd()); } -void HopperEndpoint::validate_inotify(std::pair &inotify) { - if (inotify.first == -1) - throw_errno("inotify_init"); - if (inotify.second == -1) - throw_errno("inotify_add_watch"); +std::pair HopperEndpoint::add_output_pipe(const std::filesystem::path &path) { + if (!std::filesystem::is_fifo(path)) + return std::make_pair(0, -1); + + BufferMarker *marker = m_buffer.create_marker(); + uint64_t id = next_pipe_id(0); // Type 0 for output + if (id == 0) + return std::make_pair(0, -1); + + std::cout << "OPEN OUT " << path << "\n"; + + HopperPipe *p = new HopperPipe(id, PipeType::OUT, path, marker); + m_outputs[id] = p; + + return std::make_pair(id, p->fd()); } -int HopperEndpoint::refresh(HopperDaemon *daemon) { - while (!m_operations.empty()) { - HopperEndpointOperation *op = m_operations.back(); - - int r = 0; - switch (op->type) { - case HopperEndpointOperationType::CREATE_EV: - r = daemon->add_event(op->ev); - break; - case HopperEndpointOperationType::DELETE_EV: - r = daemon->remove_event(op->ev->id); - break; +void HopperEndpoint::remove_input_pipe(const std::filesystem::path &path) { + for (const auto &[_, pipe] : m_inputs) { + if (pipe->path() == path) { + std::cout << "CLOSE IN " << path << "\n"; + m_inputs.erase(pipe->id()); + break; } - - delete op; - - m_operations.pop_back(); - - if (r != 0) - return r; } +} - return 0; +void HopperEndpoint::remove_output_pipe(const std::filesystem::path &path) { + for (const auto &[_, pipe] : m_outputs) { + if (pipe->path() == path) { + std::cout << "CLOSE OUT " << path << "\n"; + m_outputs.erase(pipe->id()); + break; + } + } } }; // namespace hopper diff --git a/daemon/meson.build b/daemon/meson.build index 1b2c352..e406b9b 100644 --- a/daemon/meson.build +++ b/daemon/meson.build @@ -4,7 +4,10 @@ executable( 'buffer.cpp', 'pipe.cpp', 'daemon.cpp', + 'daemon_endpoint.cpp', + 'daemon_inotify.cpp', 'endpoint.cpp', + 'util.cpp', include_directories: include_directories('.', '../include'), install: true, ) diff --git a/daemon/pipe.cpp b/daemon/pipe.cpp index c334fc4..9ca9cfb 100644 --- a/daemon/pipe.cpp +++ b/daemon/pipe.cpp @@ -1,20 +1,21 @@ #include +#include #include #include "hopper/daemon/pipe.hpp" +#include "hopper/daemon/util.hpp" namespace hopper { /* HopperPipe */ -HopperPipe::HopperPipe(std::string name, int handler, PipeType type, - std::filesystem::path path, BufferMarker *marker) - : m_marker(marker), m_name(name), m_handler(handler), m_type(type), - m_path(path) { - if (this->open_pipe()) - m_status = PipeStatus::ACTIVE; - else - m_status = PipeStatus::INACTIVE; +HopperPipe::HopperPipe(int id, PipeType type, std::filesystem::path path, BufferMarker *marker) + : m_marker(marker), m_type(type), m_path(path), m_id(id) { + + std::string extension = path.extension(); + std::string pipe_name = path.replace_extension("").filename(); + + open_pipe(); } HopperPipe::~HopperPipe() { @@ -29,11 +30,21 @@ int HopperPipe::open_pipe() { int fd = open(m_path.c_str(), (m_type == PipeType::IN ? O_RDONLY : O_WRONLY) | O_NONBLOCK); if (fd < 0) { - perror("open"); + if (m_type == PipeType::OUT && errno == ENXIO) { + // No readers available + m_status = PipeStatus::INACTIVE; + m_fd = -1; + return 0; + } + + m_status = PipeStatus::INACTIVE; + m_fd = -1; + throw_errno("open"); return 0; } m_fd = fd; + m_status = PipeStatus::ACTIVE; return 1; } diff --git a/daemon/util.cpp b/daemon/util.cpp new file mode 100644 index 0000000..4360d31 --- /dev/null +++ b/daemon/util.cpp @@ -0,0 +1,18 @@ +#include "hopper/daemon/util.hpp" + +namespace hopper { + +PipeType detect_pipe_type(const std::filesystem::path &path) { + if (!path.has_extension()) + return PipeType::NONE; + + if (path.extension() == ".in") + return PipeType::IN; + + if (path.extension() == ".out") + return PipeType::OUT; + + return PipeType::NONE; +} + +}; diff --git a/include/hopper/daemon/daemon.hpp b/include/hopper/daemon/daemon.hpp index d67a5d5..ae09ad6 100644 --- a/include/hopper/daemon/daemon.hpp +++ b/include/hopper/daemon/daemon.hpp @@ -2,14 +2,14 @@ #define daemon_hpp_INCLUDED #include +#include #include #include -#include -#include +#include #include "hopper/daemon/endpoint.hpp" -#include "hopper/daemon/event.hpp" +#include "hopper/daemon/pipe.hpp" namespace hopper { @@ -17,42 +17,46 @@ constexpr uint64_t INOTIFY_DATA = 0x1; class HopperDaemon { private: - std::map m_endpoints; - std::vector m_events; + std::unordered_map m_endpoints; - std::filesystem::path m_path; - uint64_t m_last_event_id = 0; - int m_last_endpoint_id = 0; + // endpoint IDs are 24 bit + uint32_t m_last_endpoint_id = 1; + uint32_t next_endpoint_id() { + if (m_last_endpoint_id > ((1ULL << 24) - 1)) + return 0; - int m_inotify_fd = -1; - int m_inotify_watch_fd = -1; + return m_last_endpoint_id++; + } + int m_inotify_fd = -1; + int m_inotify_root_watch = -1; int m_epoll_fd = -1; int m_max_events = 64; int m_timeout = 250; - uint64_t next_event_id() { return m_last_event_id++; } - int next_endpoint_id() { return m_last_endpoint_id++; } + std::filesystem::path m_path; - int create_endpoint(std::filesystem::path path); - int delete_endpoint(std::filesystem::path path); - int delete_endpoint(int id); + uint32_t create_endpoint(const std::filesystem::path &path); + void delete_endpoint(const std::filesystem::path &path); + void delete_endpoint(uint32_t id); - int handle_inotify(HopperEvent *ev); + HopperEndpoint *endpoint_by_watch(int watch); + void setup_inotify(); + void handle_inotify(); + void handle_root_inotify(struct inotify_event *ev); + void handle_endpoint_inotify(struct inotify_event *ev, HopperEndpoint *endpoint); + void process_events(struct epoll_event *events, int n_events); + void try_add_pipe(std::pair pipe, PipeType type); + void refresh_pipes(); public: HopperDaemon(std::filesystem::path path, int max_events = 64, int m_timeout = 250); ~HopperDaemon(); int run(); - - int add_event(HopperEvent *event, int events = EPOLLIN); - int remove_event(uint64_t id); - int remove_event(HopperEvent *event); - HopperEvent *get_event(uint64_t id); }; }; // namespace hopper diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp index 3248c2d..e5feb23 100644 --- a/include/hopper/daemon/endpoint.hpp +++ b/include/hopper/daemon/endpoint.hpp @@ -1,66 +1,64 @@ #ifndef endpoint_hpp_INCLUDED #define endpoint_hpp_INCLUDED -#include -#include +#include #include "hopper/daemon/buffer.hpp" -#include "hopper/daemon/event.hpp" #include "hopper/daemon/pipe.hpp" namespace hopper { -class HopperDaemon; - -enum HopperEndpointOperationType { - CREATE_EV, - DELETE_EV, -}; - -struct HopperEndpointOperation { - HopperEndpointOperationType type; - HopperEvent *ev; -}; - class HopperEndpoint { private: - std::map m_inputs; - std::map m_outputs; - std::vector m_events; - std::vector m_operations; + std::unordered_map m_inputs; + std::unordered_map m_outputs; HopperBuffer m_buffer; - std::filesystem::path m_path; - std::filesystem::path m_in_dir; - std::filesystem::path m_out_dir; + uint64_t m_last_pipe_id = 1; + uint64_t next_pipe_id(uint8_t type) { + if (m_last_pipe_id > ((1ULL << 39) - 1)) + return 0; - int m_inotify_in_fd; - int m_inotify_in_watch; + // I can say with high confidence, that we will probably + // never hit this limit. - int m_inotify_out_fd; - int m_inotify_out_watch; + // Bit mask for pipe ID (64-bit): + // EEEEEEEEEEEEEEEEEEEEEEEEPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPT + // EEE: Endpoint ID, 24-bit, ~ 16 million endpoints + // PPP: Pipe ID, 39-bit, ~ 550 billion pipes per endpoint + // T: Type, 1 for input, 0 for output + return (((uint64_t)m_id) << 40) | + ((m_last_pipe_id++ << 1) & 0xFFFFFFFFFF) | (type & 0x1); + } + std::filesystem::path m_path; std::string m_name; - int m_id; - - // First value is inotify fd, second is watch id - std::pair - create_inotify(std::filesystem::path path, - std::function callback); - void validate_inotify(std::pair &inotify); + uint32_t m_id; + int m_watch_fd; public: - HopperEndpoint(int id, std::filesystem::path path); + HopperEndpoint(uint32_t id, int watch_fd, std::filesystem::path path); + ~HopperEndpoint(); - // This is a weird fucntion. This is called by the daemon once every loop - // iteration, which then allows endpoints to manipulate daemon state such as - // updating events, etc. - int refresh(HopperDaemon *d); + void on_pipe_readable(uint64_t id); + void on_pipe_writable(uint64_t id); + + std::pair add_input_pipe(const std::filesystem::path &path); + std::pair add_output_pipe(const std::filesystem::path &path); + void remove_input_pipe(const std::filesystem::path &path); + void remove_output_pipe(const std::filesystem::path &path); - int id() { return m_id; } - const std::string &name() { return m_name; } const std::filesystem::path &path() { return m_path; } + const std::string &name() { return m_name; } + const std::unordered_map inputs() { + return m_inputs; + } + const std::unordered_map outputs() { + return m_outputs; + } + int id() { return m_id; } + int watch_fd() { return m_watch_fd; } }; }; // namespace hopper diff --git a/include/hopper/daemon/event.hpp b/include/hopper/daemon/event.hpp deleted file mode 100644 index 81f6ad1..0000000 --- a/include/hopper/daemon/event.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef event_hpp_INCLUDED -#define event_hpp_INCLUDED - -#include -#include - -namespace hopper { - -union HopperEventData { - uint64_t u64; - void *ptr; -}; - -struct HopperEvent { - int fd; - uint64_t id; - HopperEventData data; - std::function callback; -}; - -}; // namespace hopper - -#endif // event_hpp_INCLUDED diff --git a/include/hopper/daemon/pipe.hpp b/include/hopper/daemon/pipe.hpp index b72352b..c176ad9 100644 --- a/include/hopper/daemon/pipe.hpp +++ b/include/hopper/daemon/pipe.hpp @@ -9,6 +9,7 @@ namespace hopper { enum PipeType { + NONE, IN, OUT, }; @@ -23,36 +24,27 @@ class HopperPipe { BufferMarker *m_marker = nullptr; std::string m_name; - int m_handler; + PipeStatus m_status = PipeStatus::INACTIVE; PipeType m_type; - std::filesystem::path m_path; int m_fd = -1; - - PipeStatus m_status = PipeStatus::INACTIVE; + int m_id; public: - HopperPipe(std::string name, int handler, PipeType type, - std::filesystem::path path, BufferMarker *marker = nullptr); - + HopperPipe(int id, PipeType type, std::filesystem::path path, + BufferMarker *marker = nullptr); ~HopperPipe(); int open_pipe(); size_t write_pipe(void *src, size_t len); size_t read_pipe(void *dst, size_t len); - int fd() { return m_fd; } - - const std::string &name() { return m_name; } - int handler() { return m_handler; } - PipeType type() { return m_type; } - const std::filesystem::path &path() { return m_path; } - BufferMarker *marker() { return m_marker; } - PipeStatus status() { return m_status; } + int id() { return m_id; } + int fd() { return m_fd; } }; }; // namespace hopper diff --git a/include/hopper/daemon/util.hpp b/include/hopper/daemon/util.hpp index a6b535d..ef061a1 100644 --- a/include/hopper/daemon/util.hpp +++ b/include/hopper/daemon/util.hpp @@ -1,8 +1,10 @@ #ifndef util_hpp_INCLUDED #define util_hpp_INCLUDED +#include "hopper/daemon/pipe.hpp" #include #include +#include #include #include #include @@ -15,6 +17,8 @@ inline void throw_errno(const std::string &msg) { throw std::system_error(errno, std::generic_category(), ss.str()); } +PipeType detect_pipe_type(const std::filesystem::path &path); + }; // namespace hopper #endif // util_hpp_INCLUDED diff --git a/meson.build b/meson.build index 364f5eb..defae26 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'hopper', ['cpp'], version: '0.1.0', - default_options: ['optimization=2', 'warning_level=2'], + default_options: ['optimization=2', 'warning_level=2', 'cpp_std=c++20'], ) inc = include_directories('include') From 17de8aff0479975a42dc4373d5c45b96a10a403f Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Mon, 12 Jan 2026 23:45:40 +0000 Subject: [PATCH 13/46] read from pipes into endpoint buffers --- daemon/buffer.cpp | 28 +++++++++++++++++++++ daemon/daemon.cpp | 39 ++++++++++++++++++++++++------ daemon/daemon_inotify.cpp | 6 +++-- daemon/endpoint.cpp | 30 ++++++++++++++--------- daemon/pipe.cpp | 17 +++++++++---- include/hopper/daemon/buffer.hpp | 4 +-- include/hopper/daemon/daemon.hpp | 3 ++- include/hopper/daemon/endpoint.hpp | 5 ++-- include/hopper/daemon/pipe.hpp | 9 ++++--- meson.build | 3 ++- 10 files changed, 107 insertions(+), 37 deletions(-) diff --git a/daemon/buffer.cpp b/daemon/buffer.cpp index 919b3c0..0de2a52 100644 --- a/daemon/buffer.cpp +++ b/daemon/buffer.cpp @@ -1,6 +1,7 @@ #include #include "hopper/daemon/buffer.hpp" +#include "hopper/daemon/pipe.hpp" namespace hopper { @@ -45,6 +46,33 @@ size_t HopperBuffer::write(void *src, size_t len) { return done_len; } +size_t HopperBuffer::write(HopperPipe *pipe) { + size_t max_len = max_write(); + size_t done_len = 0; + + size_t next_len = std::min((m_buf.size() - m_edge), max_len); + size_t res = pipe->read_pipe(&m_buf[m_edge], next_len); + if (res == (size_t)-1) + // -1 indicates read error + return -1; + + m_edge = (m_edge + res) % m_buf.size(); + done_len += res; + if (res <= next_len) + // read_pipe reads as much as possible up to next_len, so the pipe is + // empty here + return done_len; + + next_len = max_len - next_len; + res = pipe->read_pipe(&m_buf[m_edge], next_len); + if (res == (size_t)-1) + return -1; + + m_edge = (m_edge + res) % m_buf.size(); + done_len += res; + return done_len; +} + size_t HopperBuffer::read(BufferMarker *m, void *dst, size_t len) { // Just see the arithmetic in `write`, it's the same here. diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp index dee5182..18a5148 100644 --- a/daemon/daemon.cpp +++ b/daemon/daemon.cpp @@ -34,26 +34,49 @@ HopperDaemon::~HopperDaemon() { delete endpoint; } -void HopperDaemon::try_add_pipe(std::pair pipe, PipeType type) { +void HopperDaemon::remove_pipe(HopperEndpoint *endpoint, uint64_t pipe_id) { + PipeType type = (pipe_id & 0x1 ? PipeType::IN : PipeType::OUT); + for (const auto &[id, pipe] : + (type == PipeType::IN ? endpoint->inputs() : endpoint->outputs())) { + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, pipe->fd(), nullptr) != 0) + throw_errno("epoll_ctl DEL"); + pipe->close_pipe(); + + std::cout << "DOWN " << pipe->name() << "(" << endpoint->name() + << ")\n"; + } +} + +void HopperDaemon::add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe) { + if (pipe == nullptr) + return; + // Pipe has bad ID or bad FD - if (pipe.first == 0 || pipe.second == -1) + if (pipe->id() == 0 || pipe->fd() == -1) return; struct epoll_event ev = {}; - ev.events = (type == PipeType::IN ? EPOLLIN | EPOLLHUP : EPOLLHUP); - ev.data.u64 = pipe.first; + ev.events = (pipe->type() == PipeType::IN ? EPOLLIN | EPOLLHUP : EPOLLHUP); + ev.data.u64 = pipe->id(); - if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, pipe.second, &ev) != 0) + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, pipe->fd(), &ev) != 0) throw_errno("epoll_ctl ADD"); + + std::cout << "UP " << pipe->name() << "(" << endpoint->name() << ")\n"; } void HopperDaemon::refresh_pipes() { // Try to open any inactive pipes again for (const auto &[_, endpoint] : m_endpoints) { + for (const auto &[id, pipe] : endpoint->inputs()) { + if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) + continue; + add_pipe(endpoint, pipe); + } for (const auto &[id, pipe] : endpoint->outputs()) { if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) continue; - try_add_pipe(std::make_pair(id, pipe->fd()), PipeType::OUT); + add_pipe(endpoint, pipe); } } } @@ -76,8 +99,8 @@ void HopperDaemon::process_events(struct epoll_event *events, int n_events) { if (ev.events & EPOLLIN) endpoint->on_pipe_readable(ev.data.u64); - else if (ev.events & EPOLLOUT) - endpoint->on_pipe_writable(ev.data.u64); + if (ev.events & EPOLLHUP) + remove_pipe(endpoint, ev.data.u64); } } diff --git a/daemon/daemon_inotify.cpp b/daemon/daemon_inotify.cpp index 685b64f..a577a7d 100644 --- a/daemon/daemon_inotify.cpp +++ b/daemon/daemon_inotify.cpp @@ -57,10 +57,12 @@ void HopperDaemon::handle_endpoint_inotify(struct inotify_event *ev, if (pipe_type == PipeType::NONE) return; - std::pair pipe = + HopperPipe *pipe = (pipe_type == PipeType::IN ? endpoint->add_input_pipe(p) : endpoint->add_output_pipe(p)); - try_add_pipe(pipe, pipe_type); + + if (pipe != nullptr) + add_pipe(endpoint, pipe); } } diff --git a/daemon/endpoint.cpp b/daemon/endpoint.cpp index 9dbb2fe..405cc0c 100644 --- a/daemon/endpoint.cpp +++ b/daemon/endpoint.cpp @@ -3,6 +3,7 @@ #include "hopper/daemon/endpoint.hpp" #include "hopper/daemon/pipe.hpp" +#include "hopper/daemon/util.hpp" namespace hopper { @@ -19,44 +20,49 @@ HopperEndpoint::~HopperEndpoint() { } void HopperEndpoint::on_pipe_readable(uint64_t id) { - std::cout << "ID " << id << " is readable\n"; -} + if (!m_inputs.contains(id)) + return; + + HopperPipe *pipe = m_inputs[id]; + + size_t res = m_buffer.write(pipe); + if (res == (size_t)-1) + throw_errno("read"); -void HopperEndpoint::on_pipe_writable(uint64_t id) { - std::cout << "ID " << id << " is writable\n"; + std::cout << pipe->name() << "(" << m_name << ") -> " << res << " bytes\n"; } -std::pair HopperEndpoint::add_input_pipe(const std::filesystem::path &path) { +HopperPipe *HopperEndpoint::add_input_pipe(const std::filesystem::path &path) { if (!std::filesystem::is_fifo(path)) - return std::make_pair(0, -1); + return nullptr; uint64_t id = next_pipe_id(1); // Type 1 for input if (id == 0) // ID 0 is never valid - return std::make_pair(0, -1); + return nullptr; std::cout << "OPEN IN " << path << "\n"; HopperPipe *p = new HopperPipe(id, PipeType::IN, path, nullptr); m_inputs[id] = p; - return std::make_pair(id, p->fd()); + return p; } -std::pair HopperEndpoint::add_output_pipe(const std::filesystem::path &path) { +HopperPipe *HopperEndpoint::add_output_pipe(const std::filesystem::path &path) { if (!std::filesystem::is_fifo(path)) - return std::make_pair(0, -1); + return nullptr; BufferMarker *marker = m_buffer.create_marker(); uint64_t id = next_pipe_id(0); // Type 0 for output if (id == 0) - return std::make_pair(0, -1); + return nullptr; std::cout << "OPEN OUT " << path << "\n"; HopperPipe *p = new HopperPipe(id, PipeType::OUT, path, marker); m_outputs[id] = p; - return std::make_pair(id, p->fd()); + return p; } void HopperEndpoint::remove_input_pipe(const std::filesystem::path &path) { diff --git a/daemon/pipe.cpp b/daemon/pipe.cpp index 9ca9cfb..5742631 100644 --- a/daemon/pipe.cpp +++ b/daemon/pipe.cpp @@ -9,12 +9,9 @@ namespace hopper { /* HopperPipe */ -HopperPipe::HopperPipe(int id, PipeType type, std::filesystem::path path, BufferMarker *marker) +HopperPipe::HopperPipe(uint64_t id, PipeType type, std::filesystem::path path, BufferMarker *marker) : m_marker(marker), m_type(type), m_path(path), m_id(id) { - - std::string extension = path.extension(); - std::string pipe_name = path.replace_extension("").filename(); - + m_name = path.replace_extension("").filename(); open_pipe(); } @@ -49,6 +46,16 @@ int HopperPipe::open_pipe() { return 1; } +void HopperPipe::close_pipe() { + if (m_status == PipeStatus::INACTIVE) + return; + + m_status = PipeStatus::INACTIVE; + + if (m_fd != -1) + close(m_fd); +} + size_t HopperPipe::write_pipe(void *src, size_t len) { if (m_type == PipeType::IN) return -1; diff --git a/include/hopper/daemon/buffer.hpp b/include/hopper/daemon/buffer.hpp index b6e8624..ecee355 100644 --- a/include/hopper/daemon/buffer.hpp +++ b/include/hopper/daemon/buffer.hpp @@ -24,10 +24,10 @@ class HopperBuffer { BufferMarker* create_marker(); size_t write(void *src, size_t len); - size_t write(const HopperPipe &pipe, size_t len); + size_t write(HopperPipe *pipe); size_t read(BufferMarker *marker, void *dst, size_t len); - size_t read(BufferMarker *marker, const HopperPipe &pipe, size_t len); + size_t read(HopperPipe *pipe); size_t max_write(); size_t max_read(BufferMarker *marker); diff --git a/include/hopper/daemon/daemon.hpp b/include/hopper/daemon/daemon.hpp index ae09ad6..99956e2 100644 --- a/include/hopper/daemon/daemon.hpp +++ b/include/hopper/daemon/daemon.hpp @@ -49,7 +49,8 @@ class HopperDaemon { void handle_endpoint_inotify(struct inotify_event *ev, HopperEndpoint *endpoint); void process_events(struct epoll_event *events, int n_events); - void try_add_pipe(std::pair pipe, PipeType type); + void remove_pipe(HopperEndpoint *endpoint, uint64_t pipe_id); + void add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe); void refresh_pipes(); public: HopperDaemon(std::filesystem::path path, int max_events = 64, diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp index e5feb23..0fa1dd6 100644 --- a/include/hopper/daemon/endpoint.hpp +++ b/include/hopper/daemon/endpoint.hpp @@ -42,10 +42,9 @@ class HopperEndpoint { ~HopperEndpoint(); void on_pipe_readable(uint64_t id); - void on_pipe_writable(uint64_t id); - std::pair add_input_pipe(const std::filesystem::path &path); - std::pair add_output_pipe(const std::filesystem::path &path); + HopperPipe *add_input_pipe(const std::filesystem::path &path); + HopperPipe *add_output_pipe(const std::filesystem::path &path); void remove_input_pipe(const std::filesystem::path &path); void remove_output_pipe(const std::filesystem::path &path); diff --git a/include/hopper/daemon/pipe.hpp b/include/hopper/daemon/pipe.hpp index c176ad9..de84d99 100644 --- a/include/hopper/daemon/pipe.hpp +++ b/include/hopper/daemon/pipe.hpp @@ -29,21 +29,24 @@ class HopperPipe { std::filesystem::path m_path; int m_fd = -1; - int m_id; + uint64_t m_id; public: - HopperPipe(int id, PipeType type, std::filesystem::path path, + HopperPipe(uint64_t id, PipeType type, std::filesystem::path path, BufferMarker *marker = nullptr); ~HopperPipe(); int open_pipe(); + void close_pipe(); size_t write_pipe(void *src, size_t len); size_t read_pipe(void *dst, size_t len); const std::filesystem::path &path() { return m_path; } BufferMarker *marker() { return m_marker; } PipeStatus status() { return m_status; } - int id() { return m_id; } + PipeType type() { return m_type; } + const std::string &name() { return m_name; } + uint64_t id() { return m_id; } int fd() { return m_fd; } }; diff --git a/meson.build b/meson.build index defae26..590fd3e 100644 --- a/meson.build +++ b/meson.build @@ -2,9 +2,10 @@ project( 'hopper', ['cpp'], version: '0.1.0', - default_options: ['optimization=2', 'warning_level=2', 'cpp_std=c++20'], + default_options: ['optimization=2', 'warning_level=3', 'cpp_std=c++20'], ) + inc = include_directories('include') install_headers( From 1b6f9494f3bd6fc8eecd61558d416a16f07dfe0d Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Wed, 14 Jan 2026 22:51:36 +0000 Subject: [PATCH 14/46] write data out to pipes (prev hopper parity?) --- daemon/buffer.cpp | 65 ++++++++++++++++++++++++------ daemon/daemon.cpp | 50 +---------------------- daemon/daemon_inotify.cpp | 11 +++++ daemon/daemon_pipe.cpp | 62 ++++++++++++++++++++++++++++ daemon/endpoint.cpp | 65 ++++++++++++++++++++++++------ daemon/meson.build | 1 + include/hopper/daemon/buffer.hpp | 1 + include/hopper/daemon/endpoint.hpp | 4 ++ meson.build | 2 +- 9 files changed, 185 insertions(+), 76 deletions(-) create mode 100644 daemon/daemon_pipe.cpp diff --git a/daemon/buffer.cpp b/daemon/buffer.cpp index 0de2a52..8de5014 100644 --- a/daemon/buffer.cpp +++ b/daemon/buffer.cpp @@ -1,6 +1,8 @@ +#include #include #include "hopper/daemon/buffer.hpp" +#include "hopper/daemon/marker.hpp" #include "hopper/daemon/pipe.hpp" namespace hopper { @@ -22,13 +24,19 @@ BufferMarker *HopperBuffer::create_marker() { return m; } +void HopperBuffer::delete_marker(BufferMarker *marker) { + auto tgt = std::find(m_markers.begin(), m_markers.end(), marker); + if (tgt != m_markers.end()) + m_markers.erase(tgt); +} + size_t HopperBuffer::write(void *src, size_t len) { size_t max_len = std::min(len, max_write()); size_t done_len = 0; // Write up to the buffer length, but only if len > buf_len size_t next_len = std::min((m_buf.size() - m_edge), max_len); - std::memcpy(src, &m_buf[m_edge], next_len); + std::memcpy(&m_buf[m_edge], src, next_len); m_edge = (m_edge + next_len) % m_buf.size(); // Enough bytes have been written, return @@ -38,7 +46,7 @@ size_t HopperBuffer::write(void *src, size_t len) { // We are guaranteed tp have space for whatever's left next_len = max_len - next_len; - std::memcpy(reinterpret_cast(src) + done_len, &m_buf[m_edge], + std::memcpy(&m_buf[m_edge], reinterpret_cast(src) + done_len, next_len); m_edge = (m_edge + next_len) % m_buf.size(); @@ -80,7 +88,7 @@ size_t HopperBuffer::read(BufferMarker *m, void *dst, size_t len) { size_t done_len = 0; size_t next_len = std::min((m_buf.size() - m->pos()), max_len); - std::memcpy(&m_buf[m->pos()], dst, next_len); + std::memcpy(dst, &m_buf[m->pos()], next_len); m->seek(next_len, m_buf.size(), SeekDirection::FORWARD); done_len += next_len; @@ -88,7 +96,7 @@ size_t HopperBuffer::read(BufferMarker *m, void *dst, size_t len) { return done_len; next_len = max_len - next_len; - std::memcpy(&m_buf[m->pos()], reinterpret_cast(dst) + done_len, + std::memcpy(reinterpret_cast(dst) + done_len, &m_buf[m->pos()], next_len); m->seek(next_len, m_buf.size(), SeekDirection::FORWARD); @@ -96,23 +104,54 @@ size_t HopperBuffer::read(BufferMarker *m, void *dst, size_t len) { return done_len; } +size_t HopperBuffer::read(HopperPipe *pipe) { + BufferMarker *m = pipe->marker(); + size_t max_len = max_read(m); + size_t done_len = 0; + + size_t next_len = std::min((m_buf.size() - m->pos()), max_len); + size_t res = pipe->write_pipe(&m_buf[m->pos()], next_len); + if (res == (size_t)-1) + return -1; + + m->seek(res, m_buf.size(), SeekDirection::FORWARD); + done_len += res; + if (res <= next_len) + return done_len; + + next_len = max_len - next_len; + res = pipe->write_pipe(&m_buf[m->pos()], next_len); + if (res == (size_t)-1) + return -1; + + m->seek(res, m_buf.size(), SeekDirection::FORWARD); + done_len += res; + return done_len; +} + size_t HopperBuffer::max_write() { + size_t cap = m_buf.size(); if (m_markers.empty()) - return m_buf.size(); + return cap; + + size_t min_dist = cap; + + for (auto *m : m_markers) { + size_t d = (m->pos() + cap - m_edge) % cap; - size_t min_pos = m_edge; + if (d == 0) // Marker is at m_edge, full buffer left + d = cap; - for (auto *m : m_markers) - if (m->pos() < min_pos) - min_pos = m->pos(); + if (d < min_dist) + min_dist = d; + } - return ((m_edge <= min_pos) ? min_pos - m_edge - : (m_buf.size() - m_edge) + min_pos); + return min_dist; } size_t HopperBuffer::max_read(BufferMarker *m) { - return ((m->pos() <= m_edge) ? m_edge - m->pos() - : (m_buf.size() - m->pos()) + m_edge); + size_t cap = m_buf.size(); + return (m_edge + cap - m->pos()) % cap; } }; // namespace hopper diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp index 18a5148..7b5806f 100644 --- a/daemon/daemon.cpp +++ b/daemon/daemon.cpp @@ -8,7 +8,6 @@ #include "hopper/daemon/daemon.hpp" #include "hopper/daemon/endpoint.hpp" -#include "hopper/daemon/pipe.hpp" #include "hopper/daemon/util.hpp" namespace hopper { @@ -34,53 +33,6 @@ HopperDaemon::~HopperDaemon() { delete endpoint; } -void HopperDaemon::remove_pipe(HopperEndpoint *endpoint, uint64_t pipe_id) { - PipeType type = (pipe_id & 0x1 ? PipeType::IN : PipeType::OUT); - for (const auto &[id, pipe] : - (type == PipeType::IN ? endpoint->inputs() : endpoint->outputs())) { - if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, pipe->fd(), nullptr) != 0) - throw_errno("epoll_ctl DEL"); - pipe->close_pipe(); - - std::cout << "DOWN " << pipe->name() << "(" << endpoint->name() - << ")\n"; - } -} - -void HopperDaemon::add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe) { - if (pipe == nullptr) - return; - - // Pipe has bad ID or bad FD - if (pipe->id() == 0 || pipe->fd() == -1) - return; - - struct epoll_event ev = {}; - ev.events = (pipe->type() == PipeType::IN ? EPOLLIN | EPOLLHUP : EPOLLHUP); - ev.data.u64 = pipe->id(); - - if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, pipe->fd(), &ev) != 0) - throw_errno("epoll_ctl ADD"); - - std::cout << "UP " << pipe->name() << "(" << endpoint->name() << ")\n"; -} - -void HopperDaemon::refresh_pipes() { - // Try to open any inactive pipes again - for (const auto &[_, endpoint] : m_endpoints) { - for (const auto &[id, pipe] : endpoint->inputs()) { - if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) - continue; - add_pipe(endpoint, pipe); - } - for (const auto &[id, pipe] : endpoint->outputs()) { - if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) - continue; - add_pipe(endpoint, pipe); - } - } -} - void HopperDaemon::process_events(struct epoll_event *events, int n_events) { for (int i = 0; i < n_events; i++) { struct epoll_event ev = events[i]; @@ -99,7 +51,7 @@ void HopperDaemon::process_events(struct epoll_event *events, int n_events) { if (ev.events & EPOLLIN) endpoint->on_pipe_readable(ev.data.u64); - if (ev.events & EPOLLHUP) + if (ev.events & EPOLLHUP || ev.events & EPOLLERR) remove_pipe(endpoint, ev.data.u64); } } diff --git a/daemon/daemon_inotify.cpp b/daemon/daemon_inotify.cpp index a577a7d..57edf9f 100644 --- a/daemon/daemon_inotify.cpp +++ b/daemon/daemon_inotify.cpp @@ -64,6 +64,17 @@ void HopperDaemon::handle_endpoint_inotify(struct inotify_event *ev, if (pipe != nullptr) add_pipe(endpoint, pipe); } + + if (ev->mask & IN_DELETE) { + std::filesystem::path p = endpoint->path(); + p /= ev->name; + + HopperPipe *pipe = endpoint->pipe_by_path(p); + if (pipe != nullptr && pipe->status() == PipeStatus::ACTIVE) + remove_pipe(endpoint, pipe->id()); + + endpoint->remove_by_id(pipe->id()); + } } void HopperDaemon::handle_inotify() { diff --git a/daemon/daemon_pipe.cpp b/daemon/daemon_pipe.cpp new file mode 100644 index 0000000..e3663ee --- /dev/null +++ b/daemon/daemon_pipe.cpp @@ -0,0 +1,62 @@ +#include "hopper/daemon/daemon.hpp" +#include "hopper/daemon/util.hpp" +#include + +namespace hopper { + +void HopperDaemon::remove_pipe(HopperEndpoint *endpoint, uint64_t pipe_id) { + PipeType type = (pipe_id & 0x1 ? PipeType::IN : PipeType::OUT); + for (const auto &[id, pipe] : + (type == PipeType::IN ? endpoint->inputs() : endpoint->outputs())) { + + if (pipe->id() != pipe_id) + continue; + + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_DEL, pipe->fd(), nullptr) != 0) + throw_errno("epoll_ctl DEL"); + pipe->close_pipe(); + + std::cout << "DOWN " << pipe->name() << "(" << endpoint->name() + << ")\n"; + } +} + +void HopperDaemon::add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe) { + if (pipe == nullptr) + return; + + // Pipe has bad ID or bad FD + if (pipe->id() == 0 || pipe->fd() == -1) + return; + + struct epoll_event ev = {}; + ev.events = (pipe->type() == PipeType::IN ? EPOLLIN | EPOLLHUP | EPOLLET + : EPOLLHUP); + ev.data.u64 = pipe->id(); + + if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, pipe->fd(), &ev) != 0) + throw_errno("epoll_ctl ADD"); + + std::cout << "UP " << pipe->name() << "(" << endpoint->name() << ")\n"; +} + +void HopperDaemon::refresh_pipes() { + // Try to open any inactive pipes again + for (const auto &[_, endpoint] : m_endpoints) { + for (const auto &[id, pipe] : endpoint->inputs()) { + if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) + continue; + add_pipe(endpoint, pipe); + } + for (const auto &[id, pipe] : endpoint->outputs()) { + if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) + continue; + add_pipe(endpoint, pipe); + } + + // Try to empty buffers into pipes + endpoint->flush_pipes(); + } +} + +}; // namespace hopper diff --git a/daemon/endpoint.cpp b/daemon/endpoint.cpp index 405cc0c..b4a8558 100644 --- a/daemon/endpoint.cpp +++ b/daemon/endpoint.cpp @@ -7,7 +7,8 @@ namespace hopper { -HopperEndpoint::HopperEndpoint(uint32_t id, int watch_fd, std::filesystem::path path) +HopperEndpoint::HopperEndpoint(uint32_t id, int watch_fd, + std::filesystem::path path) : m_path(path), m_id(id), m_watch_fd(watch_fd) { m_name = path.filename(); } @@ -24,7 +25,7 @@ void HopperEndpoint::on_pipe_readable(uint64_t id) { return; HopperPipe *pipe = m_inputs[id]; - + size_t res = m_buffer.write(pipe); if (res == (size_t)-1) throw_errno("read"); @@ -32,16 +33,36 @@ void HopperEndpoint::on_pipe_readable(uint64_t id) { std::cout << pipe->name() << "(" << m_name << ") -> " << res << " bytes\n"; } +void HopperEndpoint::flush_pipes() { + for (const auto &[_, pipe] : m_outputs) { + if (pipe->status() == PipeStatus::INACTIVE) + continue; + + m_buffer.read(pipe); + } +} + +HopperPipe *HopperEndpoint::pipe_by_path(const std::filesystem::path &path) { + for (const auto &[_, pipe] : m_outputs) + if (pipe->path() == path) + return pipe; + for (const auto &[_, pipe] : m_inputs) + if (pipe->path() == path) + return pipe; + + return 0; +} + HopperPipe *HopperEndpoint::add_input_pipe(const std::filesystem::path &path) { if (!std::filesystem::is_fifo(path)) return nullptr; - + uint64_t id = next_pipe_id(1); // Type 1 for input - if (id == 0) // ID 0 is never valid + if (id == 0) // ID 0 is never valid return nullptr; std::cout << "OPEN IN " << path << "\n"; - + HopperPipe *p = new HopperPipe(id, PipeType::IN, path, nullptr); m_inputs[id] = p; @@ -56,30 +77,48 @@ HopperPipe *HopperEndpoint::add_output_pipe(const std::filesystem::path &path) { uint64_t id = next_pipe_id(0); // Type 0 for output if (id == 0) return nullptr; - + std::cout << "OPEN OUT " << path << "\n"; - + HopperPipe *p = new HopperPipe(id, PipeType::OUT, path, marker); m_outputs[id] = p; return p; } +void HopperEndpoint::remove_by_id(uint64_t pipe_id) { + PipeType type = (pipe_id & 0x1 ? PipeType::IN : PipeType::OUT); + + if (type == PipeType::IN && m_inputs.contains(pipe_id)) { + HopperPipe *pipe = m_inputs[pipe_id]; + m_buffer.delete_marker(pipe->marker()); + std::cout << "CLOSE IN " << pipe->path() << "\n"; + + delete pipe; + m_inputs.erase(pipe_id); + } else if (type == PipeType::OUT && m_outputs.contains(pipe_id)) { + HopperPipe *pipe = m_outputs[pipe_id]; + m_buffer.delete_marker(pipe->marker()); + std::cout << "CLOSE OUT " << pipe->path() << "\n"; + + delete pipe; + m_outputs.erase(pipe_id); + } +} + void HopperEndpoint::remove_input_pipe(const std::filesystem::path &path) { - for (const auto &[_, pipe] : m_inputs) { + for (const auto &[id, pipe] : m_inputs) { if (pipe->path() == path) { - std::cout << "CLOSE IN " << path << "\n"; - m_inputs.erase(pipe->id()); + remove_by_id(id); break; } } } void HopperEndpoint::remove_output_pipe(const std::filesystem::path &path) { - for (const auto &[_, pipe] : m_outputs) { + for (const auto &[id, pipe] : m_outputs) { if (pipe->path() == path) { - std::cout << "CLOSE OUT " << path << "\n"; - m_outputs.erase(pipe->id()); + remove_by_id(id); break; } } diff --git a/daemon/meson.build b/daemon/meson.build index e406b9b..3cbe14c 100644 --- a/daemon/meson.build +++ b/daemon/meson.build @@ -6,6 +6,7 @@ executable( 'daemon.cpp', 'daemon_endpoint.cpp', 'daemon_inotify.cpp', + 'daemon_pipe.cpp', 'endpoint.cpp', 'util.cpp', include_directories: include_directories('.', '../include'), diff --git a/include/hopper/daemon/buffer.hpp b/include/hopper/daemon/buffer.hpp index ecee355..3e4bc0b 100644 --- a/include/hopper/daemon/buffer.hpp +++ b/include/hopper/daemon/buffer.hpp @@ -22,6 +22,7 @@ class HopperBuffer { : m_buf(len) {} // Use 1 MiB size by default BufferMarker* create_marker(); + void delete_marker(BufferMarker *marker); size_t write(void *src, size_t len); size_t write(HopperPipe *pipe); diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp index 0fa1dd6..d55ff92 100644 --- a/include/hopper/daemon/endpoint.hpp +++ b/include/hopper/daemon/endpoint.hpp @@ -42,12 +42,16 @@ class HopperEndpoint { ~HopperEndpoint(); void on_pipe_readable(uint64_t id); + void flush_pipes(); HopperPipe *add_input_pipe(const std::filesystem::path &path); HopperPipe *add_output_pipe(const std::filesystem::path &path); + void remove_by_id(uint64_t pipe_id); void remove_input_pipe(const std::filesystem::path &path); void remove_output_pipe(const std::filesystem::path &path); + HopperPipe *pipe_by_path(const std::filesystem::path &path); + const std::filesystem::path &path() { return m_path; } const std::string &name() { return m_name; } const std::unordered_map inputs() { diff --git a/meson.build b/meson.build index 590fd3e..a015f06 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'hopper', ['cpp'], version: '0.1.0', - default_options: ['optimization=2', 'warning_level=3', 'cpp_std=c++20'], + default_options: ['optimization=0', 'warning_level=3', 'cpp_std=c++20'], ) From 3cf4ae29f14d3782ce8cd5399703a3c201dd8709 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Wed, 14 Jan 2026 23:14:02 +0000 Subject: [PATCH 15/46] add pretty printing for pipe and endpoint names --- daemon/daemon.cpp | 3 --- daemon/daemon_endpoint.cpp | 4 ++-- daemon/daemon_inotify.cpp | 2 +- daemon/daemon_pipe.cpp | 11 +++++------ daemon/endpoint.cpp | 26 ++++++++++++++------------ daemon/main.cpp | 4 ++-- daemon/pipe.cpp | 8 +++++--- daemon/util.cpp | 4 ++-- include/hopper/daemon/buffer.hpp | 2 +- include/hopper/daemon/daemon.hpp | 7 ++++--- include/hopper/daemon/endpoint.hpp | 6 ++++++ include/hopper/daemon/pipe.hpp | 12 ++++++++++-- meson.build | 2 +- 13 files changed, 53 insertions(+), 38 deletions(-) diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp index 7b5806f..c97b1ce 100644 --- a/daemon/daemon.cpp +++ b/daemon/daemon.cpp @@ -59,9 +59,6 @@ void HopperDaemon::process_events(struct epoll_event *events, int n_events) { int HopperDaemon::run() { int res = 0; - std::cout << "Started event loop, max " << m_max_events - << " events, timeout " << m_timeout << " ms" << std::endl; - while (res == 0) { struct epoll_event *events = new struct epoll_event[m_max_events]; diff --git a/daemon/daemon_endpoint.cpp b/daemon/daemon_endpoint.cpp index daee4f5..dbd6386 100644 --- a/daemon/daemon_endpoint.cpp +++ b/daemon/daemon_endpoint.cpp @@ -17,7 +17,7 @@ uint32_t HopperDaemon::create_endpoint(const std::filesystem::path &path) { auto *endpoint = new HopperEndpoint(endpoint_id, inotify_watch_fd, path); m_endpoints[endpoint_id] = endpoint; - std::cout << "CREATE " << endpoint->path() << std::endl; + std::cout << "CREATE " << *endpoint << std::endl; return endpoint_id; } @@ -26,7 +26,7 @@ void HopperDaemon::delete_endpoint(uint32_t id) { if (!m_endpoints.contains(id)) return; - std::cout << "DELETE " << m_endpoints[id]->path() << std::endl; + std::cout << "DELETE " << *(m_endpoints[id]) << std::endl; delete m_endpoints[id]; m_endpoints.erase(id); diff --git a/daemon/daemon_inotify.cpp b/daemon/daemon_inotify.cpp index 57edf9f..05c8afa 100644 --- a/daemon/daemon_inotify.cpp +++ b/daemon/daemon_inotify.cpp @@ -62,7 +62,7 @@ void HopperDaemon::handle_endpoint_inotify(struct inotify_event *ev, : endpoint->add_output_pipe(p)); if (pipe != nullptr) - add_pipe(endpoint, pipe); + add_pipe(pipe); } if (ev->mask & IN_DELETE) { diff --git a/daemon/daemon_pipe.cpp b/daemon/daemon_pipe.cpp index e3663ee..7ca6043 100644 --- a/daemon/daemon_pipe.cpp +++ b/daemon/daemon_pipe.cpp @@ -16,12 +16,11 @@ void HopperDaemon::remove_pipe(HopperEndpoint *endpoint, uint64_t pipe_id) { throw_errno("epoll_ctl DEL"); pipe->close_pipe(); - std::cout << "DOWN " << pipe->name() << "(" << endpoint->name() - << ")\n"; + std::cout << "DOWN " << *pipe << "\n"; } } -void HopperDaemon::add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe) { +void HopperDaemon::add_pipe(HopperPipe *pipe) { if (pipe == nullptr) return; @@ -37,7 +36,7 @@ void HopperDaemon::add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe) { if (epoll_ctl(m_epoll_fd, EPOLL_CTL_ADD, pipe->fd(), &ev) != 0) throw_errno("epoll_ctl ADD"); - std::cout << "UP " << pipe->name() << "(" << endpoint->name() << ")\n"; + std::cout << "UP " << *pipe << "\n"; } void HopperDaemon::refresh_pipes() { @@ -46,12 +45,12 @@ void HopperDaemon::refresh_pipes() { for (const auto &[id, pipe] : endpoint->inputs()) { if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) continue; - add_pipe(endpoint, pipe); + add_pipe(pipe); } for (const auto &[id, pipe] : endpoint->outputs()) { if (pipe->status() == PipeStatus::ACTIVE || !pipe->open_pipe()) continue; - add_pipe(endpoint, pipe); + add_pipe(pipe); } // Try to empty buffers into pipes diff --git a/daemon/endpoint.cpp b/daemon/endpoint.cpp index b4a8558..4c8d022 100644 --- a/daemon/endpoint.cpp +++ b/daemon/endpoint.cpp @@ -30,7 +30,7 @@ void HopperEndpoint::on_pipe_readable(uint64_t id) { if (res == (size_t)-1) throw_errno("read"); - std::cout << pipe->name() << "(" << m_name << ") -> " << res << " bytes\n"; + std::cout << *pipe << " -> " << res << " bytes\n"; } void HopperEndpoint::flush_pipes() { @@ -38,7 +38,9 @@ void HopperEndpoint::flush_pipes() { if (pipe->status() == PipeStatus::INACTIVE) continue; - m_buffer.read(pipe); + size_t res = m_buffer.read(pipe); + if (res > 0) + std::cout << *pipe << " <- " << res << " bytes\n"; } } @@ -61,11 +63,11 @@ HopperPipe *HopperEndpoint::add_input_pipe(const std::filesystem::path &path) { if (id == 0) // ID 0 is never valid return nullptr; - std::cout << "OPEN IN " << path << "\n"; - - HopperPipe *p = new HopperPipe(id, PipeType::IN, path, nullptr); + HopperPipe *p = new HopperPipe(id, m_name, PipeType::IN, path, nullptr); m_inputs[id] = p; + std::cout << "OPEN " << *p << "\n"; + return p; } @@ -78,11 +80,11 @@ HopperPipe *HopperEndpoint::add_output_pipe(const std::filesystem::path &path) { if (id == 0) return nullptr; - std::cout << "OPEN OUT " << path << "\n"; - - HopperPipe *p = new HopperPipe(id, PipeType::OUT, path, marker); + HopperPipe *p = new HopperPipe(id, m_name, PipeType::OUT, path, marker); m_outputs[id] = p; + std::cout << "OPEN " << *p << "\n"; + return p; } @@ -92,15 +94,15 @@ void HopperEndpoint::remove_by_id(uint64_t pipe_id) { if (type == PipeType::IN && m_inputs.contains(pipe_id)) { HopperPipe *pipe = m_inputs[pipe_id]; m_buffer.delete_marker(pipe->marker()); - std::cout << "CLOSE IN " << pipe->path() << "\n"; - + std::cout << "CLOSE " << *pipe << "\n"; + delete pipe; m_inputs.erase(pipe_id); } else if (type == PipeType::OUT && m_outputs.contains(pipe_id)) { HopperPipe *pipe = m_outputs[pipe_id]; m_buffer.delete_marker(pipe->marker()); - std::cout << "CLOSE OUT " << pipe->path() << "\n"; - + std::cout << "CLOSE " << *pipe << "\n"; + delete pipe; m_outputs.erase(pipe_id); } diff --git a/daemon/main.cpp b/daemon/main.cpp index 235cec3..cd106b3 100644 --- a/daemon/main.cpp +++ b/daemon/main.cpp @@ -10,11 +10,11 @@ int main(int argc, char *argv[]) { std::cout << "Usage: hopperd " << std::endl; return 1; } else if (argc == 2) { - std::cout << "Using Hopper at " << argv[1] << std::endl; + std::cout << "HOPPER " << argv[1] << std::endl; auto daemon = hopper::HopperDaemon(argv[1]); return daemon.run(); } else if (char *p = getenv("HOPPER_PATH")) { - std::cout << "Using Hopper at " << p << std::endl; + std::cout << "HOPPER " << p << std::endl; auto daemon = hopper::HopperDaemon(p); return daemon.run(); } else { diff --git a/daemon/pipe.cpp b/daemon/pipe.cpp index 5742631..d9ebe18 100644 --- a/daemon/pipe.cpp +++ b/daemon/pipe.cpp @@ -1,5 +1,4 @@ #include -#include #include #include "hopper/daemon/pipe.hpp" @@ -9,8 +8,11 @@ namespace hopper { /* HopperPipe */ -HopperPipe::HopperPipe(uint64_t id, PipeType type, std::filesystem::path path, BufferMarker *marker) - : m_marker(marker), m_type(type), m_path(path), m_id(id) { +HopperPipe::HopperPipe(uint64_t id, const std::string &endpoint_name, + PipeType type, std::filesystem::path path, + BufferMarker *marker) + : m_marker(marker), m_type(type), m_path(path), + m_endpoint_name(endpoint_name), m_id(id) { m_name = path.replace_extension("").filename(); open_pipe(); } diff --git a/daemon/util.cpp b/daemon/util.cpp index 4360d31..f3638de 100644 --- a/daemon/util.cpp +++ b/daemon/util.cpp @@ -1,7 +1,7 @@ #include "hopper/daemon/util.hpp" namespace hopper { - + PipeType detect_pipe_type(const std::filesystem::path &path) { if (!path.has_extension()) return PipeType::NONE; @@ -15,4 +15,4 @@ PipeType detect_pipe_type(const std::filesystem::path &path) { return PipeType::NONE; } -}; +}; // namespace hopper diff --git a/include/hopper/daemon/buffer.hpp b/include/hopper/daemon/buffer.hpp index 3e4bc0b..06ead43 100644 --- a/include/hopper/daemon/buffer.hpp +++ b/include/hopper/daemon/buffer.hpp @@ -21,7 +21,7 @@ class HopperBuffer { HopperBuffer(size_t len = 1024 * 1024) : m_buf(len) {} // Use 1 MiB size by default - BufferMarker* create_marker(); + BufferMarker *create_marker(); void delete_marker(BufferMarker *marker); size_t write(void *src, size_t len); diff --git a/include/hopper/daemon/daemon.hpp b/include/hopper/daemon/daemon.hpp index 99956e2..26859da 100644 --- a/include/hopper/daemon/daemon.hpp +++ b/include/hopper/daemon/daemon.hpp @@ -19,7 +19,6 @@ class HopperDaemon { private: std::unordered_map m_endpoints; - // endpoint IDs are 24 bit uint32_t m_last_endpoint_id = 1; uint32_t next_endpoint_id() { @@ -46,12 +45,14 @@ class HopperDaemon { void setup_inotify(); void handle_inotify(); void handle_root_inotify(struct inotify_event *ev); - void handle_endpoint_inotify(struct inotify_event *ev, HopperEndpoint *endpoint); + void handle_endpoint_inotify(struct inotify_event *ev, + HopperEndpoint *endpoint); void process_events(struct epoll_event *events, int n_events); void remove_pipe(HopperEndpoint *endpoint, uint64_t pipe_id); - void add_pipe(HopperEndpoint *endpoint, HopperPipe *pipe); + void add_pipe(HopperPipe *pipe); void refresh_pipes(); + public: HopperDaemon(std::filesystem::path path, int max_events = 64, int m_timeout = 250); diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp index d55ff92..9d3a25d 100644 --- a/include/hopper/daemon/endpoint.hpp +++ b/include/hopper/daemon/endpoint.hpp @@ -62,6 +62,12 @@ class HopperEndpoint { } int id() { return m_id; } int watch_fd() { return m_watch_fd; } + + friend std::ostream &operator<<(std::ostream &os, + HopperEndpoint &endpoint) { + os << endpoint.m_name << "(" << endpoint.m_id << ")"; + return os; + } }; }; // namespace hopper diff --git a/include/hopper/daemon/pipe.hpp b/include/hopper/daemon/pipe.hpp index de84d99..d733a6c 100644 --- a/include/hopper/daemon/pipe.hpp +++ b/include/hopper/daemon/pipe.hpp @@ -28,12 +28,14 @@ class HopperPipe { PipeType m_type; std::filesystem::path m_path; + const std::string &m_endpoint_name; + int m_fd = -1; uint64_t m_id; public: - HopperPipe(uint64_t id, PipeType type, std::filesystem::path path, - BufferMarker *marker = nullptr); + HopperPipe(uint64_t id, const std::string &endpoint_name, PipeType type, + std::filesystem::path path, BufferMarker *marker = nullptr); ~HopperPipe(); int open_pipe(); @@ -48,6 +50,12 @@ class HopperPipe { const std::string &name() { return m_name; } uint64_t id() { return m_id; } int fd() { return m_fd; } + + friend std::ostream &operator<<(std::ostream &os, const HopperPipe &pipe) { + os << (pipe.m_type == PipeType::IN ? "+" : "-") << pipe.m_name << "(" + << pipe.m_endpoint_name << ")"; + return os; + }; }; }; // namespace hopper diff --git a/meson.build b/meson.build index a015f06..590fd3e 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'hopper', ['cpp'], version: '0.1.0', - default_options: ['optimization=0', 'warning_level=3', 'cpp_std=c++20'], + default_options: ['optimization=2', 'warning_level=3', 'cpp_std=c++20'], ) From 69e48a453a20e1e89fb274796a6661fb305e28f5 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 15:16:09 +0000 Subject: [PATCH 16/46] fix: add nullptr check in pipe removal to prevent segfault --- daemon/buffer.cpp | 4 ++-- daemon/daemon_inotify.cpp | 9 ++++++--- flake.nix | 4 ++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/daemon/buffer.cpp b/daemon/buffer.cpp index 8de5014..3b67e77 100644 --- a/daemon/buffer.cpp +++ b/daemon/buffer.cpp @@ -137,7 +137,7 @@ size_t HopperBuffer::max_write() { size_t min_dist = cap; for (auto *m : m_markers) { - size_t d = (m->pos() + cap - m_edge) % cap; + size_t d = (m->pos() - m_edge + cap) % cap; if (d == 0) // Marker is at m_edge, full buffer left d = cap; @@ -151,7 +151,7 @@ size_t HopperBuffer::max_write() { size_t HopperBuffer::max_read(BufferMarker *m) { size_t cap = m_buf.size(); - return (m_edge + cap - m->pos()) % cap; + return (m_edge - m->pos() + cap) % cap; } }; // namespace hopper diff --git a/daemon/daemon_inotify.cpp b/daemon/daemon_inotify.cpp index 05c8afa..7cbbc2e 100644 --- a/daemon/daemon_inotify.cpp +++ b/daemon/daemon_inotify.cpp @@ -70,10 +70,13 @@ void HopperDaemon::handle_endpoint_inotify(struct inotify_event *ev, p /= ev->name; HopperPipe *pipe = endpoint->pipe_by_path(p); - if (pipe != nullptr && pipe->status() == PipeStatus::ACTIVE) - remove_pipe(endpoint, pipe->id()); - endpoint->remove_by_id(pipe->id()); + if (pipe != nullptr) { + if (pipe->status() == PipeStatus::ACTIVE) + remove_pipe(endpoint, pipe->id()); + + endpoint->remove_by_id(pipe->id()); + } } } diff --git a/flake.nix b/flake.nix index 9a44c1b..ae1bf19 100644 --- a/flake.nix +++ b/flake.nix @@ -43,6 +43,10 @@ meson ninja pkg-config + + # one probably wants these too + gdb + valgrind ]; }; } From 34f99eda61501ebfc07d3561401bbe97a1b54d64 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 15:58:33 +0000 Subject: [PATCH 17/46] fix various memory issues - fix uninitialised memory use in `buffer.cpp` - experiment with smart pointers in `daemon.cpp` --- daemon/buffer.cpp | 2 ++ daemon/daemon.cpp | 9 +++++---- include/hopper/daemon/buffer.hpp | 6 +++--- include/hopper/daemon/endpoint.hpp | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/daemon/buffer.cpp b/daemon/buffer.cpp index 3b67e77..5612ea6 100644 --- a/daemon/buffer.cpp +++ b/daemon/buffer.cpp @@ -18,6 +18,8 @@ void BufferMarker::seek(size_t offset, size_t max, SeekDirection dir) { /* HopperBuffer */ +HopperBuffer::HopperBuffer(size_t len) : m_edge(0) { m_buf.resize(len); } + BufferMarker *HopperBuffer::create_marker() { auto *m = new BufferMarker(m_edge); m_markers.push_back(m); diff --git a/daemon/daemon.cpp b/daemon/daemon.cpp index c97b1ce..37fa2c5 100644 --- a/daemon/daemon.cpp +++ b/daemon/daemon.cpp @@ -59,20 +59,21 @@ void HopperDaemon::process_events(struct epoll_event *events, int n_events) { int HopperDaemon::run() { int res = 0; + std::unique_ptr events( + new struct epoll_event[m_max_events]); + while (res == 0) { - struct epoll_event *events = new struct epoll_event[m_max_events]; - int n = epoll_wait(m_epoll_fd, events, m_max_events, m_timeout); + int n = epoll_wait(m_epoll_fd, events.get(), m_max_events, m_timeout); if (n < 0) { if (errno == EINTR) continue; - delete[] events; throw_errno("epoll_wait"); return -1; } - process_events(events, n); + process_events(events.get(), n); refresh_pipes(); } diff --git a/include/hopper/daemon/buffer.hpp b/include/hopper/daemon/buffer.hpp index 06ead43..8e61f52 100644 --- a/include/hopper/daemon/buffer.hpp +++ b/include/hopper/daemon/buffer.hpp @@ -1,6 +1,7 @@ #ifndef buffer_hpp_INCLUDED #define buffer_hpp_INCLUDED +#include #include #include "hopper/daemon/marker.hpp" @@ -12,14 +13,13 @@ class HopperPipe; class HopperBuffer { private: - std::vector m_buf; + std::vector m_buf; std::vector m_markers; size_t m_edge; public: - HopperBuffer(size_t len = 1024 * 1024) - : m_buf(len) {} // Use 1 MiB size by default + HopperBuffer(size_t len = 1024 * 1024); // Use 1 MiB size by default BufferMarker *create_marker(); void delete_marker(BufferMarker *marker); diff --git a/include/hopper/daemon/endpoint.hpp b/include/hopper/daemon/endpoint.hpp index 9d3a25d..e11d067 100644 --- a/include/hopper/daemon/endpoint.hpp +++ b/include/hopper/daemon/endpoint.hpp @@ -13,7 +13,7 @@ class HopperEndpoint { std::unordered_map m_inputs; std::unordered_map m_outputs; - HopperBuffer m_buffer; + HopperBuffer m_buffer{}; uint64_t m_last_pipe_id = 1; uint64_t next_pipe_id(uint8_t type) { From 42726fefad07754a10eb5a1b28b76704202c4a1a Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 16:19:30 +0000 Subject: [PATCH 18/46] use C instead of C++ for client library --- client/lib.c | 5 +++++ client/lib.cpp | 7 ------- client/meson.build | 2 +- include/hopper/hopper.h | 8 -------- meson.build | 2 +- 5 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 client/lib.c delete mode 100644 client/lib.cpp diff --git a/client/lib.c b/client/lib.c new file mode 100644 index 0000000..d4d70b0 --- /dev/null +++ b/client/lib.c @@ -0,0 +1,5 @@ +#include + +#include "hopper/hopper.h" + +void hello() { printf("hello world\n"); } diff --git a/client/lib.cpp b/client/lib.cpp deleted file mode 100644 index 25c70da..0000000 --- a/client/lib.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include - -#include "hopper/hopper.h" - -extern "C" { -void hello() { std::cout << "hello world\n" << std::endl; } -} diff --git a/client/meson.build b/client/meson.build index 0d086ed..db9317c 100644 --- a/client/meson.build +++ b/client/meson.build @@ -2,7 +2,7 @@ client_inc = include_directories('.', '../include') libhopper = library( 'hopper', - 'lib.cpp', + 'lib.c', include_directories: client_inc, version: meson.project_version(), install: true, diff --git a/include/hopper/hopper.h b/include/hopper/hopper.h index 12bbf2f..6fcb19c 100644 --- a/include/hopper/hopper.h +++ b/include/hopper/hopper.h @@ -1,14 +1,6 @@ #ifndef hopper_h_INCLUDED #define hopper_h_INCLUDED -#ifdef __cplusplus -extern "C" { -#endif - void hello(); -#ifdef __cplusplus -} -#endif - #endif // hopper_h_INCLUDED diff --git a/meson.build b/meson.build index 590fd3e..284dd7e 100644 --- a/meson.build +++ b/meson.build @@ -1,6 +1,6 @@ project( 'hopper', - ['cpp'], + ['c', 'cpp'], version: '0.1.0', default_options: ['optimization=2', 'warning_level=3', 'cpp_std=c++20'], ) From 2a53966b48524dda5892aa477cac8d008023c880 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 18:26:57 +0000 Subject: [PATCH 19/46] feat: add initial `libhopper` client library --- client/lib.c | 119 +++++++++++++++++++++++++++++++++++++++- include/hopper/hopper.h | 33 ++++++++++- 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/client/lib.c b/client/lib.c index d4d70b0..28352e4 100644 --- a/client/lib.c +++ b/client/lib.c @@ -1,5 +1,122 @@ +#include +#include +#include #include +#include +#include +#include +#include #include "hopper/hopper.h" -void hello() { printf("hello world\n"); } +int get_open_flags(int flags) { + int open_flags = 0; + + if (flags & HOPPER_IN) + open_flags |= O_WRONLY; + if (flags & HOPPER_OUT) + open_flags |= O_RDONLY; + if (flags & HOPPER_NONBLOCK) + open_flags |= O_NONBLOCK; + + return open_flags; +} + +char *get_pipe_path(struct hopper_pipe *pipe) { + char *path = (char *)malloc(sizeof(char) * PATH_MAX); + const char *suffix = (pipe->flags & HOPPER_IN ? "in" : "out"); + + // {HOPPER}/{ENDPOINT}/{NAME}.{TYPE} + sprintf(path, "%s/%s/%s.%s", pipe->hopper, pipe->endpoint, pipe->name, + suffix); + + return path; +} + +int hopper_open(struct hopper_pipe *pipe) { + int res = 0; + + if (pipe->name == NULL || pipe->endpoint == NULL) { + // pipe and endpoint name are (obviously) required + errno = EINVAL; + return -1; + } + + if ((pipe->flags & HOPPER_IN && pipe->flags & HOPPER_OUT) || + (pipe->flags & ~HOPPER_IN && pipe->flags & ~HOPPER_OUT)) { + // either both or none of the input/output flags are set + errno = EINVAL; + return -1; + } + + if (pipe->hopper == NULL) { + // hopper path isn't overridden, get from environment + pipe->hopper = getenv("HOPPER_PATH"); + + if (pipe->hopper == NULL) { + // still don't have a valid path, can't open pipe + errno = ENOENT; + return -1; + } + } + + char *pipe_path = get_pipe_path(pipe); + if (mknod(pipe_path, S_IFIFO | 0660, 0) < 0 && errno != EEXIST) { + // mknod failed in some way, preserve errno + res = -1; + goto cleanup; + } + + int open_flags = get_open_flags(pipe->flags); + int fd = open(pipe_path, open_flags); + if (fd < 0) { + // preserve errno if open fails + res = -1; + goto cleanup; + } + + // acquire a file lock on the fifo, we don't want other things using it + if (flock(fd, (pipe->flags & HOPPER_IN ? LOCK_EX : LOCK_SH) | LOCK_NB) != + 0) { + if (errno == EWOULDBLOCK) + errno = EBUSY; // this makes more sense for clients + + res = -1; + close(fd); + goto cleanup; + } + + pipe->fd = fd; + +cleanup: + free(pipe_path); + if (res != 0) + pipe->fd = -1; + + return res; +} + +void hopper_close(struct hopper_pipe *pipe) { + if (pipe->fd == -1) + return; + + flock(pipe->fd, LOCK_UN); + close(pipe->fd); + + pipe->fd = -1; +} + +ssize_t hopper_read(struct hopper_pipe *pipe, void *dst, size_t len) { + ssize_t res = read(pipe->fd, dst, len); + if (res < 0 && errno == EWOULDBLOCK) + return 0; // EWOULDBLOCK isn't an error for non-block pipes + return res; +} + +ssize_t hopper_write(struct hopper_pipe *pipe, void *src, size_t len) { + ssize_t res = write(pipe->fd, src, len); + if (res < 0 && errno == EWOULDBLOCK) + return 0; // EWOULDBLOCK isn't an error for non-block pipes + return res; +} + diff --git a/include/hopper/hopper.h b/include/hopper/hopper.h index 6fcb19c..fe9d44a 100644 --- a/include/hopper/hopper.h +++ b/include/hopper/hopper.h @@ -1,6 +1,37 @@ #ifndef hopper_h_INCLUDED #define hopper_h_INCLUDED -void hello(); +#include + +#define HOPPER_IN 1 +#define HOPPER_OUT 2 +#define HOPPER_NONBLOCK 4 + +/// Structure representing a Hopper pipe +struct hopper_pipe { + const char *name; + const char *endpoint; + const char *hopper; + int fd; + int flags; +}; + +/// Open a new Hopper pipe specified by `pipe`. +/// Hopper location will be determined from the `HOPPER_PATH` environment +/// variable, or can be overridden with the `hopper_pipe.hopper` string. +/// -1 is returned on error, and errno is set. +int hopper_open(struct hopper_pipe *pipe); + +/// Close a Hopper pipe previously opened by `hopper_open_pipe`. +void hopper_close(struct hopper_pipe *pipe); + +/// Read up to `len` bytes from a Hopper pipe. Value returned indicates +/// the number of bytes read. -1 is returned on error and errno is set. +ssize_t hopper_read(struct hopper_pipe *pipe, void *dst, size_t len); + +/// Write up to `len` bytes into a Hopper pipe. Value returned indicates +/// the number of bytes written. -1 is returned on error and errno is set. +ssize_t hopper_write(struct hopper_pipe *pipe, void *src, size_t len); #endif // hopper_h_INCLUDED + From 20dbb5c191b35751d2a96da73883026af52098da Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 18:30:44 +0000 Subject: [PATCH 20/46] client: don't expose helper functions --- client/lib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/lib.c b/client/lib.c index 28352e4..79a5179 100644 --- a/client/lib.c +++ b/client/lib.c @@ -9,7 +9,7 @@ #include "hopper/hopper.h" -int get_open_flags(int flags) { +static int get_open_flags(int flags) { int open_flags = 0; if (flags & HOPPER_IN) @@ -22,7 +22,7 @@ int get_open_flags(int flags) { return open_flags; } -char *get_pipe_path(struct hopper_pipe *pipe) { +static char *get_pipe_path(struct hopper_pipe *pipe) { char *path = (char *)malloc(sizeof(char) * PATH_MAX); const char *suffix = (pipe->flags & HOPPER_IN ? "in" : "out"); From b92f255ac0c13142462e69e4ed6eeb1f157b6b92 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 18:32:22 +0000 Subject: [PATCH 21/46] daemon: replace EAGAIN with EWOULDBLOCK EWOULDBLOCK is specifically for non-blocking I/O, they just happen to be the same on Linux, don't rely on this. --- daemon/pipe.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/pipe.cpp b/daemon/pipe.cpp index d9ebe18..3e3db2c 100644 --- a/daemon/pipe.cpp +++ b/daemon/pipe.cpp @@ -71,7 +71,7 @@ size_t HopperPipe::write_pipe(void *src, size_t len) { ssize_t res = write(m_fd, reinterpret_cast(src) + done_len, len - done_len); - if (res == -1 && (errno == EAGAIN || errno == EINTR)) + if (res == -1 && (errno == EWOULDBLOCK || errno == EINTR)) return done_len; else if (res == -1) { perror("write"); @@ -97,7 +97,7 @@ size_t HopperPipe::read_pipe(void *dst, size_t len) { ssize_t res = read(m_fd, reinterpret_cast(dst) + done_len, len - done_len); - if (res == -1 && (errno == EAGAIN || errno == EINTR)) + if (res == -1 && (errno == EWOULDBLOCK || errno == EINTR)) return done_len; else if (res == -1) { perror("read"); From 1bb584df55c19c89db426f08210b286111071bba Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 18:45:19 +0000 Subject: [PATCH 22/46] client: link as static library, not shared library --- client/meson.build | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/meson.build b/client/meson.build index db9317c..7084459 100644 --- a/client/meson.build +++ b/client/meson.build @@ -1,10 +1,10 @@ client_inc = include_directories('.', '../include') -libhopper = library( +libhopper = static_library( 'hopper', 'lib.c', include_directories: client_inc, - version: meson.project_version(), + pic: true, install: true, ) @@ -14,6 +14,7 @@ pkg.generate( libhopper, name: 'hopper', description: 'Hopper client library', + version: meson.project_version(), subdirs: 'hopper', ) From 90e7b2da05ac165d5848dcfb6f9842f53801d07c Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 19:07:37 +0000 Subject: [PATCH 23/46] client: preserve errno across close syscall in flock error path --- client/lib.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/lib.c b/client/lib.c index 79a5179..c00be65 100644 --- a/client/lib.c +++ b/client/lib.c @@ -81,8 +81,11 @@ int hopper_open(struct hopper_pipe *pipe) { if (errno == EWOULDBLOCK) errno = EBUSY; // this makes more sense for clients - res = -1; + int errsv = errno; // preserve errno across close close(fd); + errno = errsv; + + res = -1; goto cleanup; } From 568c4cf3a589df15164582baded29ad9eda50979 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Thu, 15 Jan 2026 19:10:15 +0000 Subject: [PATCH 24/46] client: replace mknod call with mkfifo --- client/lib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/lib.c b/client/lib.c index c00be65..92c45ac 100644 --- a/client/lib.c +++ b/client/lib.c @@ -61,8 +61,8 @@ int hopper_open(struct hopper_pipe *pipe) { } char *pipe_path = get_pipe_path(pipe); - if (mknod(pipe_path, S_IFIFO | 0660, 0) < 0 && errno != EEXIST) { - // mknod failed in some way, preserve errno + if (mkfifo(pipe_path, 0660) < 0 && errno != EEXIST) { + // mkfifo failed in some way, preserve errno res = -1; goto cleanup; } From 702b8156074d06080a27ae374c2637932398c9c4 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 17 Jan 2026 15:07:11 +0000 Subject: [PATCH 25/46] clean up README.md --- README.md | 192 ++---------------------------------------------------- 1 file changed, 5 insertions(+), 187 deletions(-) diff --git a/README.md b/README.md index 2de9ca2..0ae8757 100644 --- a/README.md +++ b/README.md @@ -1,191 +1,9 @@ # hopper -Pipe multiplexer for internal communication. +Hopper is a general-purpose, stream-oriented, broadcast IPC system for *nix +systems. -Hopper is the core part of the new communication system that ties together -various systems within the (updated) RoboCon brains. Hopper is only relevant -in brains for 2026 (and perhaps later) competitions. Where applicable, Hopper -is a primarily dependency for Shepherd and the robot library, but is also -needed for other smaller services. In the future, Hopper will be a dependency -of Wardog (unfinished as of 2026). +## License -Hopper's primary purpose is to provide a multiplexer for named pipes, -specifically, FIFOs. I (OldUser101, Nathan Gill) may use the terms -"FIFO", "pipe", and "named pipe" interchangeably. If you really want to, -go and research the differences between these terms. In simple terms, -Hopper is a service that acts as a one-way broadcaster between various -data channels. In even simpler terms, one program can send a message, and -any number of other programs which are listening for it, can get the message. -It is functionally similar to MQTT (which was originally discussed prior to -Hopper's design in late 2025, and Hopper's inspiration), but doesn't touch -the network stack. The advantage of this is lower latency compared to the -network-based MQTT. The primary disadvantage is that external clients -(outside the brain) cannot communicate with the system directly, adding -network overhead and complexity for, say, the arena. - -Originally, Hopper was written completely in Python, mainly for -interoperability with other services such as Shepherd, which (at the time of -writing) is also written in Python. Hopper is formed of multiple modules. -Each module of Hopper will be explained briefly, and in detail later. - -The first part is the `server` module, which is a standalone program -that handles the multiplexing between named pipes in a directory. Due to -potential latency issues, and the sheer overhead of Python, the server module -was rewritten in C, during the development of the (still unfinished) Wardog -hardware server. - -The second is the `client` module, which provides Python wrappers around Hopper -communication. In effect, it provides basic functions to allow Python clients -to open, read, write, and close named pipes in such a way that the Hopper -server can use them. - -The third module is named `common`. This module provides common **Python**, -functionality, such as named pipe handling. Ironically, since the rewrite of -the Hopper server, the `common` module only provides functionality for the -`client` module, rather than being shared, as the name suggests. - -Finally, the `util` module consists purely of programs that can be used to -test the functionality of both the Hopper `server`, and `client` modules. -Like `common`, it is badly named, and should probably be called `tests`, -but I can't be bothered to rename it. - -## `server` - -The `server` module is the standalone pipe multiplexer server that facilitates -Hopper itself. It is (now) written in C. If you don't know POSIX and/or Linux -APIs in general, this section may seem complicated. This section focuses on the -current C-based Hopper server, which improves upon the Python version (which is -largely undocumented). The core focus of the rewrite was significantly reducing -the latency when sending/reciving data through the server. This is critical for -both system stability, and the planned Wardog hardware server, which requires -near-zero latency for precise hardware timing. - -In the Python version, -a single message would have an average latency of around 0.25 seconds, for -data buffers <1 KiB. Above this size, delays of seconds or longer could occur. -This was not only due to the overhead of Python, but also inefficiencies and -filesystem constraints. The primary part of this was that pipes were opened -in non-blocking mode, since the server was single threaded, and blocking is -non-ideal. Because of this, a 0.25 second delay was added to prevent the -process from consuming all system resources, constantly, and crashing Shepherd. - -**OUT OF DATE**:The way the current Hopper server achieves this near-zero latency, and low -system resource usage, involves the use of Linux and POSIX APIs that are not -trivially exposed in Python. More specifically, `epoll` is used to block -the server when no data needs to be transferred, while keeping file descriptors -open in non-blocking mode for other purposes; the use of `splice`, `tee`, and -intermediate pipes mean that the transferred data buffers are never copied -into user space, allowing the kernel to effectively manage them. In effect, -the Hopper server doesn't actually copy any data itself, reducing latency -from the copy process, and allowing for the 1 KiB limit to be increased to -1 MiB in a single transfer. - -**UPDATED**: Due to complexities and issues with using kernel buffers, a simple -ring buffer is used instead, which is slightly slower, but probably fine -for out use case. - -For coordinating which FIFOs need to receive what data, Hopper uses a filename -format system. - -`I/O__` - -- `I/O`, input or output pipe. This is relative to the server, and is a little -counterintuitive. An client providing an "input" pipe is actually *sending* -through the Hopper server, not receiving it. - -- ``, the handler ID. This should really be called the channel -identifier, as it controls the group of FIFOs that data is shared between. - -- ``, a unique name for the client. - -For example, a FIFO with name `O_log_helper`, is an output pipe, that will -receive data from the Hopper server. It will receive all data from FIFOs that -have the matching handler `log` (for log messages), e.g. `I_log_robot` (logs -sent from usercode, the robot library). It is given the unique name `helper`. -This name corresponds to the `helper.py` service, provided alongside Shepherd. -As the name suggests, this FIFO will recieve logs, which need to be sent to -Sheep, and the arena. - -Neither the handler ID or client name can contain underscores, as it messes -with the format system. However, this may not be checked by the rewritten -server? - -The handler ID was originally given its name in the older Python server. -In that system, a handler was a particular class that processed data -sent through a pipe, invoked by the server. These handlers could manipulate -the data in transit, such as appending timestamps, or saving to a separate -file. The handler concept was ditched in the server rewrite, but the name, -and old values, still remain. - -Handler IDs, and their respective internal mappings are defined in -`handler.h`, and `handler.c`. - -A better way to identify channels of FIFOs could be a directory structure -like this: - -``` -/home/pi/pipes -| -----log -| | -| ----in -| | | -| | ----robot -| | | -| | ----runner -| | | -| | ... -| | -| ----out -| | -| ----helper -| | -| ----shepherd -| | -| ... -| -... -``` - -Rather than using filenames, the standard directory based model could be used. -This is much cleaner, and probably easier to understand, than the current -model. - -The new Hopper server also uses `inotify` to detect when FIFOs and directories -change, as well as other mechanisms. - -If you couldn't figure out already, the Hopper server takes a single argument: - -- ``, a path pointing to the directory of FIFOs to multiplex. - -In practice, this is only ever `/home/pi/pipes`. It you ever get a fault where -the brain boots, the Wi-Fi network is operational, but doesn't get to flashy, -and Shepherd doesn't work, try SSHing and creating this directory, as -Hopper doesn't create it, and the failure of Hopper will prevent anything else -from starting at all. - -## `client` - -The second module of Hopper is the Python client bindings. These are relatively -easy to use. `hopper` should be installed as a Python module available to -use. If not, a `setup.py` is provided. - -The client APIs are relatively easy to use, and basic examples can be found in -the `read.py` and `write.py` files in the `util` module. - -The API also provides a `JsonReader` class, which is used in the development -version of Wardog, and simplifies reading JSON from Hopper. If you want -examples of this, check out Wardog. - -## `common` - -This module provides functionality to the Hopper `client`. This was previously -used by the `server` module, before the rewrite. - -The `common` module is almost entirely re-exported by the `client` module, -so you should look at the documentation for that instead. - -## `util` - -The `util` module is (hopefully) not something you need documentation for, if -you've read everything above. +Hopper is licensed under the BSD 2-Clause license, see [LICENSE](./LICENSE) +for details. From 1f1f9dcd2b7a71c03aa539288065fbcd55464448 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 17 Jan 2026 22:09:49 +0000 Subject: [PATCH 26/46] add intial Python module structure --- client/{ => lib}/lib.c | 0 client/lib/meson.build | 20 ++++++++++++ client/meson.build | 22 ++------------ client/py/hoppermodule.c | 66 ++++++++++++++++++++++++++++++++++++++++ client/py/meson.build | 11 +++++++ flake.nix | 2 ++ nix/package.nix | 3 +- 7 files changed, 103 insertions(+), 21 deletions(-) rename client/{ => lib}/lib.c (100%) create mode 100644 client/lib/meson.build create mode 100644 client/py/hoppermodule.c create mode 100644 client/py/meson.build diff --git a/client/lib.c b/client/lib/lib.c similarity index 100% rename from client/lib.c rename to client/lib/lib.c diff --git a/client/lib/meson.build b/client/lib/meson.build new file mode 100644 index 0000000..05c12c0 --- /dev/null +++ b/client/lib/meson.build @@ -0,0 +1,20 @@ +client_inc = include_directories('.', '../../include') + +libhopper = static_library( + 'hopper', + 'lib.c', + include_directories: client_inc, + pic: true, + install: true, +) + +pkg = import('pkgconfig') + +pkg.generate( + libhopper, + name: 'hopper', + description: 'Hopper client library', + version: meson.project_version(), + subdirs: 'hopper', +) + diff --git a/client/meson.build b/client/meson.build index 7084459..b7bd3a3 100644 --- a/client/meson.build +++ b/client/meson.build @@ -1,20 +1,2 @@ -client_inc = include_directories('.', '../include') - -libhopper = static_library( - 'hopper', - 'lib.c', - include_directories: client_inc, - pic: true, - install: true, -) - -pkg = import('pkgconfig') - -pkg.generate( - libhopper, - name: 'hopper', - description: 'Hopper client library', - version: meson.project_version(), - subdirs: 'hopper', -) - +subdir('lib') +subdir('py') diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c new file mode 100644 index 0000000..9bc673b --- /dev/null +++ b/client/py/hoppermodule.c @@ -0,0 +1,66 @@ +#define PY_SSIZE_T_CLEAN +#include + +#include "hopper/hopper.h" + +// clang-format off +struct _hopper_pipe { + PyObject_HEAD + PyObject *name; + PyObject *endpoint; + PyObject *hopper; + int fd; + int flags; +}; + +static PyTypeObject _py_hopper_pipe = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "hopper.HopperPipe", + .tp_doc = PyDoc_STR("A object representing a Hopper pipe"), + .tp_basicsize = sizeof(struct _hopper_pipe), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, +}; +// clang-format on + +static PyObject *hopper_open_pipe(PyObject *self, PyObject *args) {} +static PyObject *hopper_close_pipe(PyObject *self, PyObject *args) {} +static PyObject *hopper_read_pipe(PyObject *self, PyObject *args) {} +static PyObject *hopper_write_pipe(PyObject *self, PyObject *args) {} + +static int hopper_module_exec(PyObject *m) { + if (PyType_Ready(&_py_hopper_pipe) < 0) + return -1; + + if (PyModule_AddObjectRef(m, "HopperPipe", (PyObject *)&_py_hopper_pipe) < + 0) + return -1; + + return 0; +} + +static PyMethodDef hopper_methods[] = { + {"open", hopper_open_pipe, METH_VARARGS, ""}, + {"close", hopper_close_pipe, METH_VARARGS, ""}, + {"read", hopper_read_pipe, METH_VARARGS, ""}, + {"write", hopper_write_pipe, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL}, +}; + +static PyModuleDef_Slot hopper_slots[] = { + {Py_mod_exec, (void *)hopper_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL}, +}; + +static struct PyModuleDef hoppermodule = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "hopper", + .m_doc = NULL, + .m_methods = hopper_methods, + .m_slots = hopper_slots, +}; + +PyMODINIT_FUNC PyInit_hopper(void) { return PyModuleDef_Init(&hoppermodule); } + diff --git a/client/py/meson.build b/client/py/meson.build new file mode 100644 index 0000000..6c6e38d --- /dev/null +++ b/client/py/meson.build @@ -0,0 +1,11 @@ +client_inc = include_directories('.', '../../include') + +py = import('python').find_installation('python3') + +py.extension_module( + 'hopper', + 'hoppermodule.c', + include_directories: client_inc, + install: true, +) + diff --git a/flake.nix b/flake.nix index ae1bf19..d8b5b98 100644 --- a/flake.nix +++ b/flake.nix @@ -44,6 +44,8 @@ ninja pkg-config + python313 + # one probably wants these too gdb valgrind diff --git a/nix/package.nix b/nix/package.nix index 3813b80..1fd5aee 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -1,4 +1,4 @@ -{ stdenv, meson, ninja, pkg-config }: +{ stdenv, meson, ninja, pkg-config, python313 }: stdenv.mkDerivation { name = "hopper"; @@ -9,5 +9,6 @@ stdenv.mkDerivation { meson ninja pkg-config + python313 ]; } From 7da7e211a8d65987d8b346acb1c305223c21256f Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 17 Jan 2026 23:14:10 +0000 Subject: [PATCH 27/46] client(lib): require `hopper` to be specified in `hopper_pipe` struct --- client/lib/lib.c | 15 ++------------- include/hopper/hopper.h | 6 ++---- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/client/lib/lib.c b/client/lib/lib.c index 92c45ac..968f771 100644 --- a/client/lib/lib.c +++ b/client/lib/lib.c @@ -36,8 +36,8 @@ static char *get_pipe_path(struct hopper_pipe *pipe) { int hopper_open(struct hopper_pipe *pipe) { int res = 0; - if (pipe->name == NULL || pipe->endpoint == NULL) { - // pipe and endpoint name are (obviously) required + if (pipe->name == NULL || pipe->endpoint == NULL || pipe->hopper == NULL) { + // pipe, endpoint name, hopper path are (obviously) required errno = EINVAL; return -1; } @@ -49,17 +49,6 @@ int hopper_open(struct hopper_pipe *pipe) { return -1; } - if (pipe->hopper == NULL) { - // hopper path isn't overridden, get from environment - pipe->hopper = getenv("HOPPER_PATH"); - - if (pipe->hopper == NULL) { - // still don't have a valid path, can't open pipe - errno = ENOENT; - return -1; - } - } - char *pipe_path = get_pipe_path(pipe); if (mkfifo(pipe_path, 0660) < 0 && errno != EEXIST) { // mkfifo failed in some way, preserve errno diff --git a/include/hopper/hopper.h b/include/hopper/hopper.h index fe9d44a..07829ea 100644 --- a/include/hopper/hopper.h +++ b/include/hopper/hopper.h @@ -16,10 +16,8 @@ struct hopper_pipe { int flags; }; -/// Open a new Hopper pipe specified by `pipe`. -/// Hopper location will be determined from the `HOPPER_PATH` environment -/// variable, or can be overridden with the `hopper_pipe.hopper` string. -/// -1 is returned on error, and errno is set. +/// Open a new Hopper pipe specified by `pipe`. -1 is returned on error, and +/// errno is set. int hopper_open(struct hopper_pipe *pipe); /// Close a Hopper pipe previously opened by `hopper_open_pipe`. From b80ab3ed95a314f84ea56e9d8be50d752b17dc38 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 17 Jan 2026 23:45:55 +0000 Subject: [PATCH 28/46] implement core of the `HopperPipe` type --- client/py/hoppermodule.c | 106 +++++++++++++++++++++--- client/py/meson.build | 3 +- include/hopper/client/py/hoppermodule.h | 43 ++++++++++ 3 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 include/hopper/client/py/hoppermodule.h diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index 9bc673b..1b1e2ae 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -1,26 +1,106 @@ #define PY_SSIZE_T_CLEAN #include +#include + #include "hopper/hopper.h" +#include "hoppermodule.h" // clang-format off -struct _hopper_pipe { - PyObject_HEAD - PyObject *name; - PyObject *endpoint; - PyObject *hopper; - int fd; - int flags; +Hopper_Pipe_GET(name) +Hopper_Pipe_GET(endpoint) +Hopper_Pipe_GET(hopper) +Hopper_Pipe_SETSTR(name) +Hopper_Pipe_SETSTR(endpoint) +Hopper_Pipe_SETSTR(hopper); +// clang-format on + +static PyObject *_hopper_pipe_new(PyTypeObject *type, PyObject *args, + PyObject *kwds) { + struct _hopper_pipe *self = (struct _hopper_pipe *)type->tp_alloc(type, 0); + if (!self) + return NULL; + + // Create each field with default values + // This is __new__ in Python + self->name = PyUnicode_FromString(""); + self->endpoint = PyUnicode_FromString(""); + self->hopper = PyUnicode_FromString(""); + + if (!self->name || !self->endpoint || !self->hopper) { + Py_XDECREF(self->name); + Py_XDECREF(self->endpoint); + Py_XDECREF(self->hopper); + return NULL; + } + + self->fd = -1; + self->flags = 0; + + return (PyObject *)self; +} + +static int _hopper_pipe_init(struct _hopper_pipe *self, PyObject *args, + PyObject *kwds) { + static char *kwlist[] = {"name", "endpoint", "hopper", "fd", "flags", NULL}; + PyObject *name = NULL, *endpoint = NULL, *hopper = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUUii", kwlist, &name, + &endpoint, &hopper, &self->fd, + &self->flags)) + return -1; + + // After parsing, copy the provided value into the struct + if (name) + Py_SETREF(self->name, name); + if (endpoint) + Py_SETREF(self->endpoint, endpoint); + if (hopper) + Py_SETREF(self->hopper, hopper); + + return 0; +} + +static void _hopper_pipe_dealloc(struct _hopper_pipe *self) { + assert(self->name); + assert(self->endpoint); + assert(self->hopper); + + Py_XDECREF(self->name); + Py_XDECREF(self->endpoint); + Py_XDECREF(self->hopper); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMemberDef _hopper_pipe_members[] = { + {"fd", Py_T_INT, offsetof(struct _hopper_pipe, fd), 0, + "pipe file descriptor"}, + {"flags", Py_T_INT, offsetof(struct _hopper_pipe, flags), 0, "pipe flags"}, + {NULL}, }; -static PyTypeObject _py_hopper_pipe = { +static PyGetSetDef _hopper_get_set[] = { + {"name", (getter)_hopper_pipe_getname, (setter)_hopper_pipe_setname, + "pipe name", NULL}, + {"endpoint", (getter)_hopper_pipe_getendpoint, + (setter)_hopper_pipe_setendpoint, "pipe endpoint", NULL}, + {"hopper", (getter)_hopper_pipe_gethopper, (setter)_hopper_pipe_sethopper, + "pipe hopper", NULL}, +}; + +// clang-format off +static PyTypeObject _hopper_pipe_object = { .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "hopper.HopperPipe", .tp_doc = PyDoc_STR("A object representing a Hopper pipe"), .tp_basicsize = sizeof(struct _hopper_pipe), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = PyType_GenericNew, + .tp_new = _hopper_pipe_new, + .tp_init = (initproc)_hopper_pipe_init, + .tp_dealloc = (destructor)_hopper_pipe_dealloc, + .tp_members = _hopper_pipe_members, + .tp_getset = _hopper_get_set, }; // clang-format on @@ -30,11 +110,11 @@ static PyObject *hopper_read_pipe(PyObject *self, PyObject *args) {} static PyObject *hopper_write_pipe(PyObject *self, PyObject *args) {} static int hopper_module_exec(PyObject *m) { - if (PyType_Ready(&_py_hopper_pipe) < 0) + if (PyType_Ready(&_hopper_pipe_object) < 0) return -1; - if (PyModule_AddObjectRef(m, "HopperPipe", (PyObject *)&_py_hopper_pipe) < - 0) + if (PyModule_AddObjectRef(m, "HopperPipe", + (PyObject *)&_hopper_pipe_object) < 0) return -1; return 0; @@ -49,7 +129,7 @@ static PyMethodDef hopper_methods[] = { }; static PyModuleDef_Slot hopper_slots[] = { - {Py_mod_exec, (void *)hopper_module_exec}, + {Py_mod_exec, hopper_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, {0, NULL}, }; diff --git a/client/py/meson.build b/client/py/meson.build index 6c6e38d..91b8377 100644 --- a/client/py/meson.build +++ b/client/py/meson.build @@ -1,10 +1,11 @@ -client_inc = include_directories('.', '../../include') +client_inc = include_directories('.', '../../include', '../../include/hopper/client/py') py = import('python').find_installation('python3') py.extension_module( 'hopper', 'hoppermodule.c', + c_args: ['-Wno-unused-parameter', '-Wno-pedantic'], include_directories: client_inc, install: true, ) diff --git a/include/hopper/client/py/hoppermodule.h b/include/hopper/client/py/hoppermodule.h new file mode 100644 index 0000000..e1405d9 --- /dev/null +++ b/include/hopper/client/py/hoppermodule.h @@ -0,0 +1,43 @@ +#ifndef hoppermodule_h_INCLUDED +#define hoppermodule_h_INCLUDED + +#define PY_SSIZE_T_CLEAN +#include + +// clang-format off +struct _hopper_pipe { + PyObject_HEAD + PyObject *name; + PyObject *endpoint; + PyObject *hopper; + int fd; + int flags; +}; +// clang-format on + +// These macros are horrible, but hard to replace :( +#define Hopper_Pipe_GET(field) \ + static PyObject *_hopper_pipe_get##field(struct _hopper_pipe *self, \ + void *closure) { \ + return Py_NewRef(self->field); \ + } + +#define Hopper_Pipe_SETSTR(field) \ + static int _hopper_pipe_set##field(struct _hopper_pipe *self, \ + PyObject *value, void *closure) { \ + if (!value) { \ + PyErr_SetString(PyExc_TypeError, "cannot delete " #field); \ + return -1; \ + } \ + \ + if (!PyUnicode_Check(value)) { \ + PyErr_SetString(PyExc_TypeError, \ + #field " must be a Unicode string"); \ + return -1; \ + } \ + \ + Py_SETREF(self->field, Py_NewRef(value)); \ + return 0; \ + } + +#endif // hoppermodule_h_INCLUDED From 3ec152a45182dbe943ec7ddfbb6d2257b6332534 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 17 Jan 2026 23:54:42 +0000 Subject: [PATCH 29/46] client(py): move hopper client methods onto hopper pipe object --- client/py/hoppermodule.c | 70 +++++++++++++------------ include/hopper/client/py/hoppermodule.h | 6 +-- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index 1b1e2ae..a3c3997 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -15,9 +15,10 @@ Hopper_Pipe_SETSTR(endpoint) Hopper_Pipe_SETSTR(hopper); // clang-format on -static PyObject *_hopper_pipe_new(PyTypeObject *type, PyObject *args, - PyObject *kwds) { - struct _hopper_pipe *self = (struct _hopper_pipe *)type->tp_alloc(type, 0); +static PyObject *hopper_pipe_new(PyTypeObject *type, PyObject *args, + PyObject *kwds) { + struct py_hopper_pipe *self = + (struct py_hopper_pipe *)type->tp_alloc(type, 0); if (!self) return NULL; @@ -40,8 +41,8 @@ static PyObject *_hopper_pipe_new(PyTypeObject *type, PyObject *args, return (PyObject *)self; } -static int _hopper_pipe_init(struct _hopper_pipe *self, PyObject *args, - PyObject *kwds) { +static int hopper_pipe_init(struct py_hopper_pipe *self, PyObject *args, + PyObject *kwds) { static char *kwlist[] = {"name", "endpoint", "hopper", "fd", "flags", NULL}; PyObject *name = NULL, *endpoint = NULL, *hopper = NULL; @@ -61,7 +62,7 @@ static int _hopper_pipe_init(struct _hopper_pipe *self, PyObject *args, return 0; } -static void _hopper_pipe_dealloc(struct _hopper_pipe *self) { +static void hopper_pipe_dealloc(struct py_hopper_pipe *self) { assert(self->name); assert(self->endpoint); assert(self->hopper); @@ -72,14 +73,20 @@ static void _hopper_pipe_dealloc(struct _hopper_pipe *self) { Py_TYPE(self)->tp_free((PyObject *)self); } -static PyMemberDef _hopper_pipe_members[] = { - {"fd", Py_T_INT, offsetof(struct _hopper_pipe, fd), 0, +static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) {} +static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) {} +static PyObject *hopper_pipe_read(PyObject *self, PyObject *args) {} +static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) {} + +static PyMemberDef hopper_pipe_members[] = { + {"fd", Py_T_INT, offsetof(struct py_hopper_pipe, fd), 0, "pipe file descriptor"}, - {"flags", Py_T_INT, offsetof(struct _hopper_pipe, flags), 0, "pipe flags"}, + {"flags", Py_T_INT, offsetof(struct py_hopper_pipe, flags), 0, + "pipe flags"}, {NULL}, }; -static PyGetSetDef _hopper_get_set[] = { +static PyGetSetDef hopper_pipe_get_set[] = { {"name", (getter)_hopper_pipe_getname, (setter)_hopper_pipe_setname, "pipe name", NULL}, {"endpoint", (getter)_hopper_pipe_getendpoint, @@ -88,46 +95,42 @@ static PyGetSetDef _hopper_get_set[] = { "pipe hopper", NULL}, }; +static PyMethodDef hopper_pipe_methods[] = { + {"open", hopper_pipe_open, METH_VARARGS, ""}, + {"close", hopper_pipe_close, METH_VARARGS, ""}, + {"read", hopper_pipe_read, METH_VARARGS, ""}, + {"write", hopper_pipe_write, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL}, +}; + // clang-format off -static PyTypeObject _hopper_pipe_object = { +static PyTypeObject hopper_pipe_type = { .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "hopper.HopperPipe", .tp_doc = PyDoc_STR("A object representing a Hopper pipe"), - .tp_basicsize = sizeof(struct _hopper_pipe), + .tp_basicsize = sizeof(struct py_hopper_pipe), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = _hopper_pipe_new, - .tp_init = (initproc)_hopper_pipe_init, - .tp_dealloc = (destructor)_hopper_pipe_dealloc, - .tp_members = _hopper_pipe_members, - .tp_getset = _hopper_get_set, + .tp_new = hopper_pipe_new, + .tp_init = (initproc)hopper_pipe_init, + .tp_dealloc = (destructor)hopper_pipe_dealloc, + .tp_members = hopper_pipe_members, + .tp_getset = hopper_pipe_get_set, + .tp_methods = hopper_pipe_methods, }; // clang-format on -static PyObject *hopper_open_pipe(PyObject *self, PyObject *args) {} -static PyObject *hopper_close_pipe(PyObject *self, PyObject *args) {} -static PyObject *hopper_read_pipe(PyObject *self, PyObject *args) {} -static PyObject *hopper_write_pipe(PyObject *self, PyObject *args) {} - static int hopper_module_exec(PyObject *m) { - if (PyType_Ready(&_hopper_pipe_object) < 0) + if (PyType_Ready(&hopper_pipe_type) < 0) return -1; - if (PyModule_AddObjectRef(m, "HopperPipe", - (PyObject *)&_hopper_pipe_object) < 0) + if (PyModule_AddObjectRef(m, "HopperPipe", (PyObject *)&hopper_pipe_type) < + 0) return -1; return 0; } -static PyMethodDef hopper_methods[] = { - {"open", hopper_open_pipe, METH_VARARGS, ""}, - {"close", hopper_close_pipe, METH_VARARGS, ""}, - {"read", hopper_read_pipe, METH_VARARGS, ""}, - {"write", hopper_write_pipe, METH_VARARGS, ""}, - {NULL, NULL, 0, NULL}, -}; - static PyModuleDef_Slot hopper_slots[] = { {Py_mod_exec, hopper_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, @@ -138,7 +141,6 @@ static struct PyModuleDef hoppermodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "hopper", .m_doc = NULL, - .m_methods = hopper_methods, .m_slots = hopper_slots, }; diff --git a/include/hopper/client/py/hoppermodule.h b/include/hopper/client/py/hoppermodule.h index e1405d9..469359c 100644 --- a/include/hopper/client/py/hoppermodule.h +++ b/include/hopper/client/py/hoppermodule.h @@ -5,7 +5,7 @@ #include // clang-format off -struct _hopper_pipe { +struct py_hopper_pipe { PyObject_HEAD PyObject *name; PyObject *endpoint; @@ -17,13 +17,13 @@ struct _hopper_pipe { // These macros are horrible, but hard to replace :( #define Hopper_Pipe_GET(field) \ - static PyObject *_hopper_pipe_get##field(struct _hopper_pipe *self, \ + static PyObject *_hopper_pipe_get##field(struct py_hopper_pipe *self, \ void *closure) { \ return Py_NewRef(self->field); \ } #define Hopper_Pipe_SETSTR(field) \ - static int _hopper_pipe_set##field(struct _hopper_pipe *self, \ + static int _hopper_pipe_set##field(struct py_hopper_pipe *self, \ PyObject *value, void *closure) { \ if (!value) { \ PyErr_SetString(PyExc_TypeError, "cannot delete " #field); \ From d7170292c896546822888253e4a8f207e4cd7396 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 00:20:19 +0000 Subject: [PATCH 30/46] client(py): add initial impl. of `close` function --- client/py/hoppermodule.c | 8 +++++++- client/py/meson.build | 1 + include/hopper/client/py/hoppermodule.h | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index a3c3997..23556fb 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -74,7 +74,13 @@ static void hopper_pipe_dealloc(struct py_hopper_pipe *self) { } static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) {} -static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) {} + +static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { + Hopper_Pipe_CONVERT(self, pipe); + hopper_close(&pipe); + Py_RETURN_NONE; +} + static PyObject *hopper_pipe_read(PyObject *self, PyObject *args) {} static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) {} diff --git a/client/py/meson.build b/client/py/meson.build index 91b8377..962f58b 100644 --- a/client/py/meson.build +++ b/client/py/meson.build @@ -7,6 +7,7 @@ py.extension_module( 'hoppermodule.c', c_args: ['-Wno-unused-parameter', '-Wno-pedantic'], include_directories: client_inc, + link_with: libhopper, install: true, ) diff --git a/include/hopper/client/py/hoppermodule.h b/include/hopper/client/py/hoppermodule.h index 469359c..1ef1671 100644 --- a/include/hopper/client/py/hoppermodule.h +++ b/include/hopper/client/py/hoppermodule.h @@ -40,4 +40,19 @@ struct py_hopper_pipe { return 0; \ } +#define Hopper_Pipe_CONVERT(in, out) \ + struct py_hopper_pipe *_self = (struct py_hopper_pipe *)in; \ + \ + const char *name = PyUnicode_AsUTF8(_self->name); \ + const char *endpoint = PyUnicode_AsUTF8(_self->endpoint); \ + const char *hopper = PyUnicode_AsUTF8(_self->hopper); \ + \ + struct hopper_pipe out = { \ + .name = name, \ + .endpoint = endpoint, \ + .hopper = hopper, \ + .fd = _self->fd, \ + .flags = _self->flags, \ + }; + #endif // hoppermodule_h_INCLUDED From 3a89bcb24319227ac3729634ba4781c2fbd83d9e Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 10:42:24 +0000 Subject: [PATCH 31/46] client(py): create ref.s for passed strings in `init` to avoid use-after-free --- client/py/hoppermodule.c | 8 +++++--- include/hopper/client/py/hoppermodule.h | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index 23556fb..ce0a040 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -53,11 +53,11 @@ static int hopper_pipe_init(struct py_hopper_pipe *self, PyObject *args, // After parsing, copy the provided value into the struct if (name) - Py_SETREF(self->name, name); + Py_SETREF(self->name, Py_NewRef(name)); if (endpoint) - Py_SETREF(self->endpoint, endpoint); + Py_SETREF(self->endpoint, Py_NewRef(endpoint)); if (hopper) - Py_SETREF(self->hopper, hopper); + Py_SETREF(self->hopper, Py_NewRef(hopper)); return 0; } @@ -78,6 +78,7 @@ static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) {} static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { Hopper_Pipe_CONVERT(self, pipe); hopper_close(&pipe); + Hopper_Pipe_UPDATE(pipe, self); Py_RETURN_NONE; } @@ -99,6 +100,7 @@ static PyGetSetDef hopper_pipe_get_set[] = { (setter)_hopper_pipe_setendpoint, "pipe endpoint", NULL}, {"hopper", (getter)_hopper_pipe_gethopper, (setter)_hopper_pipe_sethopper, "pipe hopper", NULL}, + {NULL}, }; static PyMethodDef hopper_pipe_methods[] = { diff --git a/include/hopper/client/py/hoppermodule.h b/include/hopper/client/py/hoppermodule.h index 1ef1671..740ab96 100644 --- a/include/hopper/client/py/hoppermodule.h +++ b/include/hopper/client/py/hoppermodule.h @@ -55,4 +55,9 @@ struct py_hopper_pipe { .flags = _self->flags, \ }; +#define Hopper_Pipe_UPDATE(in, out) \ + struct py_hopper_pipe *_out = (struct py_hopper_pipe *)out; \ + _out->fd = in.fd; \ + _out->flags = in.flags; + #endif // hoppermodule_h_INCLUDED From 43972c2e661ddcc92b92d4945fd6810f21c5e598 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 10:50:19 +0000 Subject: [PATCH 32/46] client(lib): return status code on `hopper_close` --- client/lib/lib.c | 13 +++++++++---- include/hopper/hopper.h | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/lib/lib.c b/client/lib/lib.c index 968f771..183780d 100644 --- a/client/lib/lib.c +++ b/client/lib/lib.c @@ -88,14 +88,19 @@ int hopper_open(struct hopper_pipe *pipe) { return res; } -void hopper_close(struct hopper_pipe *pipe) { +int hopper_close(struct hopper_pipe *pipe) { if (pipe->fd == -1) - return; + return 0; - flock(pipe->fd, LOCK_UN); - close(pipe->fd); + if (flock(pipe->fd, LOCK_UN) != 0) + return -1; + + if (close(pipe->fd) != 0) + return -1; pipe->fd = -1; + + return 0; } ssize_t hopper_read(struct hopper_pipe *pipe, void *dst, size_t len) { diff --git a/include/hopper/hopper.h b/include/hopper/hopper.h index 07829ea..f34bc86 100644 --- a/include/hopper/hopper.h +++ b/include/hopper/hopper.h @@ -21,7 +21,7 @@ struct hopper_pipe { int hopper_open(struct hopper_pipe *pipe); /// Close a Hopper pipe previously opened by `hopper_open_pipe`. -void hopper_close(struct hopper_pipe *pipe); +int hopper_close(struct hopper_pipe *pipe); /// Read up to `len` bytes from a Hopper pipe. Value returned indicates /// the number of bytes read. -1 is returned on error and errno is set. From acd47d4b8113251ff59cda272412f0ca32131772 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 11:01:41 +0000 Subject: [PATCH 33/46] client(py): define a `HopperError` exception type --- client/py/hoppermodule.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index ce0a040..f0866ac 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -6,6 +6,9 @@ #include "hopper/hopper.h" #include "hoppermodule.h" +// define error type for hopper things +static PyObject *HopperError = NULL; + // clang-format off Hopper_Pipe_GET(name) Hopper_Pipe_GET(endpoint) @@ -77,7 +80,11 @@ static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) {} static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { Hopper_Pipe_CONVERT(self, pipe); - hopper_close(&pipe); + + int res = hopper_close(&pipe); + if (res != 0) + return PyErr_SetFromErrno(HopperError); + Hopper_Pipe_UPDATE(pipe, self); Py_RETURN_NONE; } @@ -129,6 +136,17 @@ static PyTypeObject hopper_pipe_type = { // clang-format on static int hopper_module_exec(PyObject *m) { + if (HopperError != NULL) { + PyErr_SetString(PyExc_ImportError, + "cannot initialize hopper module multiple times"); + return -1; + } + + HopperError = PyErr_NewException("hopper.error", NULL, NULL); + + if (PyModule_AddObjectRef(m, "HopperError", HopperError) < 0) + return -1; + if (PyType_Ready(&hopper_pipe_type) < 0) return -1; From 72c5ed6b6fb3e9a88be903b426dcd9466ebc2d5d Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 11:12:06 +0000 Subject: [PATCH 34/46] client(lib): correctly check in/out flag combination in `hopper_open` --- client/lib/lib.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/lib/lib.c b/client/lib/lib.c index 183780d..9532b58 100644 --- a/client/lib/lib.c +++ b/client/lib/lib.c @@ -42,8 +42,7 @@ int hopper_open(struct hopper_pipe *pipe) { return -1; } - if ((pipe->flags & HOPPER_IN && pipe->flags & HOPPER_OUT) || - (pipe->flags & ~HOPPER_IN && pipe->flags & ~HOPPER_OUT)) { + if (!(pipe->flags & HOPPER_IN) == !(pipe->flags & HOPPER_OUT)) { // either both or none of the input/output flags are set errno = EINVAL; return -1; From cb4b9fe8ce60fa04f5828339d0bc96e95815787f Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 11:18:36 +0000 Subject: [PATCH 35/46] client(py): implement `hopper_pipe_open` function --- client/py/hoppermodule.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index f0866ac..7935f03 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -76,7 +76,17 @@ static void hopper_pipe_dealloc(struct py_hopper_pipe *self) { Py_TYPE(self)->tp_free((PyObject *)self); } -static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) {} +static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) { + Hopper_Pipe_CONVERT(self, pipe); + + int res = hopper_open(&pipe); + if (res != 0) { + return PyErr_SetFromErrno(HopperError); + } + + Hopper_Pipe_UPDATE(pipe, self); + Py_RETURN_NONE; +} static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { Hopper_Pipe_CONVERT(self, pipe); @@ -111,8 +121,8 @@ static PyGetSetDef hopper_pipe_get_set[] = { }; static PyMethodDef hopper_pipe_methods[] = { - {"open", hopper_pipe_open, METH_VARARGS, ""}, - {"close", hopper_pipe_close, METH_VARARGS, ""}, + {"open", hopper_pipe_open, METH_VARARGS, "open a Hopper pipe"}, + {"close", hopper_pipe_close, METH_VARARGS, "close a Hopper pipe"}, {"read", hopper_pipe_read, METH_VARARGS, ""}, {"write", hopper_pipe_write, METH_VARARGS, ""}, {NULL, NULL, 0, NULL}, From 1609926c5bce5e1b9319a0d22b9d432496134e3b Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 11:33:39 +0000 Subject: [PATCH 36/46] client(py): implement `hopper_pipe_write` function --- client/py/hoppermodule.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index 7935f03..eaa245b 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -100,7 +100,26 @@ static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { } static PyObject *hopper_pipe_read(PyObject *self, PyObject *args) {} -static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) {} + +static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) { + PyObject *_src = NULL; + if (!PyArg_ParseTuple(args, "O!", &PyBytes_Type, &_src)) + return NULL; + + char *src; + Py_ssize_t len; + if (PyBytes_AsStringAndSize(_src, &src, &len) != 0) + return NULL; + + Hopper_Pipe_CONVERT(self, pipe); + + ssize_t res = hopper_write(&pipe, (void *)src, (size_t)len); + if (res < 0) + return PyErr_SetFromErrno(HopperError); + + Hopper_Pipe_UPDATE(pipe, self); + return PyLong_FromSsize_t((Py_ssize_t)res); +} static PyMemberDef hopper_pipe_members[] = { {"fd", Py_T_INT, offsetof(struct py_hopper_pipe, fd), 0, From bb3d3c2a4d47a9f2ab8850394e7b06a17038b406 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 11:53:37 +0000 Subject: [PATCH 37/46] client(py): implement `hopper_pipe_read` function --- client/py/hoppermodule.c | 41 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index eaa245b..f1b12ce 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -81,6 +81,7 @@ static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) { int res = hopper_open(&pipe); if (res != 0) { + Hopper_Pipe_UPDATE(pipe, self); return PyErr_SetFromErrno(HopperError); } @@ -92,14 +93,46 @@ static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { Hopper_Pipe_CONVERT(self, pipe); int res = hopper_close(&pipe); - if (res != 0) + if (res != 0) { + Hopper_Pipe_UPDATE(pipe, self); return PyErr_SetFromErrno(HopperError); + } Hopper_Pipe_UPDATE(pipe, self); Py_RETURN_NONE; } -static PyObject *hopper_pipe_read(PyObject *self, PyObject *args) {} +static PyObject *hopper_pipe_read(PyObject *self, PyObject *args) { + Py_ssize_t len; + if (!PyArg_ParseTuple(args, "n", &len)) + return NULL; + + PyObject *_dst = PyByteArray_FromStringAndSize(NULL, len); + if (!_dst) + return NULL; + + char *dst = PyByteArray_AS_STRING(_dst); + + Hopper_Pipe_CONVERT(self, pipe); + + ssize_t res = hopper_read(&pipe, (void *)dst, (size_t)len); + if (res < 0) { + Py_DECREF(_dst); + Hopper_Pipe_UPDATE(pipe, self); + return PyErr_SetFromErrno(HopperError); + } + + if (res < len) { + if (PyByteArray_Resize(_dst, (Py_ssize_t)res) < 0) { + Py_DECREF(_dst); + Hopper_Pipe_UPDATE(pipe, self); + return NULL; + } + } + + Hopper_Pipe_UPDATE(pipe, self); + return _dst; +} static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) { PyObject *_src = NULL; @@ -114,8 +147,10 @@ static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) { Hopper_Pipe_CONVERT(self, pipe); ssize_t res = hopper_write(&pipe, (void *)src, (size_t)len); - if (res < 0) + if (res < 0) { + Hopper_Pipe_UPDATE(pipe, self); return PyErr_SetFromErrno(HopperError); + } Hopper_Pipe_UPDATE(pipe, self); return PyLong_FromSsize_t((Py_ssize_t)res); From bf7eb3db10b63fc7bbe94717d3dd82fdb3bc49f2 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 11:57:13 +0000 Subject: [PATCH 38/46] client(py): add docstrings for `hopper_read_pipe` and `hopper_write_pipe` --- client/py/hoppermodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c index f1b12ce..08a1060 100644 --- a/client/py/hoppermodule.c +++ b/client/py/hoppermodule.c @@ -177,8 +177,8 @@ static PyGetSetDef hopper_pipe_get_set[] = { static PyMethodDef hopper_pipe_methods[] = { {"open", hopper_pipe_open, METH_VARARGS, "open a Hopper pipe"}, {"close", hopper_pipe_close, METH_VARARGS, "close a Hopper pipe"}, - {"read", hopper_pipe_read, METH_VARARGS, ""}, - {"write", hopper_pipe_write, METH_VARARGS, ""}, + {"read", hopper_pipe_read, METH_VARARGS, "read bytes from a Hopper pipe"}, + {"write", hopper_pipe_write, METH_VARARGS, "write bytes to a Hopper pipe"}, {NULL, NULL, 0, NULL}, }; From f1c8e0a2cff19915d10bb1666bc67399ac8d07e1 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sun, 18 Jan 2026 12:01:03 +0000 Subject: [PATCH 39/46] add correction in README.md pretend I remembered `epoll` is Linux only --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ae8757..9a05d42 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # hopper -Hopper is a general-purpose, stream-oriented, broadcast IPC system for *nix +Hopper is a general-purpose, stream-oriented, broadcast IPC system for Linux systems systems. ## License From c2fa2b20ecdca2bf6de6b931242e39002fc1f5dd Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Tue, 20 Jan 2026 18:59:03 +0000 Subject: [PATCH 40/46] use a standard Python library, CPython extensions are hard to cross-compile --- client/meson.build | 1 - client/py/__init__.py | 7 + client/py/__pycache__/hopper.cpython-313.pyc | Bin 0 -> 5042 bytes client/py/hopper.py | 90 +++++++ client/py/hoppermodule.c | 238 ------------------- client/py/meson.build | 13 - include/hopper/client/py/hoppermodule.h | 63 ----- 7 files changed, 97 insertions(+), 315 deletions(-) create mode 100644 client/py/__init__.py create mode 100644 client/py/__pycache__/hopper.cpython-313.pyc create mode 100644 client/py/hopper.py delete mode 100644 client/py/hoppermodule.c delete mode 100644 client/py/meson.build delete mode 100644 include/hopper/client/py/hoppermodule.h diff --git a/client/meson.build b/client/meson.build index b7bd3a3..c2f563b 100644 --- a/client/meson.build +++ b/client/meson.build @@ -1,2 +1 @@ subdir('lib') -subdir('py') diff --git a/client/py/__init__.py b/client/py/__init__.py new file mode 100644 index 0000000..87b6cae --- /dev/null +++ b/client/py/__init__.py @@ -0,0 +1,7 @@ +from .hopper import HopperPipe, HopperPipeType + +__all__ = [ + "HopperPipe", + "HopperPipeType", +] + diff --git a/client/py/__pycache__/hopper.cpython-313.pyc b/client/py/__pycache__/hopper.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9d04104352288313d9d0c48fb8e3046f6a4a55a GIT binary patch literal 5042 zcmdTIU2Ie5^}B!mPn-lh7!QaWQoti6WetK9DC5T=G?A0~#v_pfS59&hOr7iWyAFto zA+$+V!l)9eMuZ6oG;LCOY9Bjk(&z-!`qX4*igg=JNZTHG^9WO`^01xrU0>T6v+i|A z$@#wTobUJi`>xm3xd_0u2bt6*Cn0~sfd#f=XMGSli$o?e*Gs6`a+G6Zp7PN0y~5if z6+`poFJGkDC%LfwF%F_(&JZKo%+#34GBH;itZ#?$#jw9Rg{So)wAi8rfL~r^CzQ*&>tT({EDJyliAdm zlFVjlYIG*6>WV@^Vq_ifY3%Pfo0(EOv}E>dvV*GA8NEZlsAttF6{e2?6YtkCmQJZ! zwqyEYk+6IEB9&k{LwIT31Ly;ich+4wns<4BevCF+?B|5z1}egO8qo8?*-GRlnI8cG zjevmQUxI(zh^^Bl+k@nsc#flX*%2hvAv*y(Wfx$V>;~+XJtLk@UarG=v`(%csfUr5 zjl6OLZJ2BfZ!lbCVL!%u7-AM=ZipyE!=Y;9)0vc(HEiaa4TqM|M$?(GcgHMy!i%F7 z2=Koi1+++p8t@?|429B`?#a;W>7u<5GW6p#u3#`UkS^L50jrFYG)j@dBnk7;$NvTh z+h{Y8=te{VM4J!=0U3faF%EVk>S}u8rL%OraEJkI!c|~HDJ_)+!`M)k!ETF)s?%Em zT_KNrfosk@LxIq>x_tBIYwkSjZSc>EA3B&;$4ak+H6?ijfC|3GY4EBTFsdO@l1Hp0 zf%2R;hX_gf2#IrvlC)!vGT-XG26Hs7f6w=x!J>n;%52#g8>-I0$5(-Q2Aoh|#em_? zA}LwDPs59Ii~DeGbBq zyS?*Wb6vOUR|DadKse_P=RJ)d$8IF9CssV6xk$dQaejDi_}1>#VEan2Jy+LW$)Anj z`a|=FOycS5r&m0!K=k@%JujLI1|H@<_o1^$5`ayJmCVDU5E3gnN0|m{DH+If)EdSD zb8DDEJs4@C&RSbr&9NHl+*>sJdh&*_x9HGq=(Gm68x*?@b~MCkh@*yh4g!hkw9I|c zkqtr5Ql@=FI-g9>sCPN0OdWSED$&W5$hPoRs}HFyFjapG&=s=gZJfU_cVYh0+@+=C zw|j5)e&YLd%iS%XcmBTXw_Ts_y7$g~{ZE}=b$!|O)!{D>=T0P+yGNGypU!#DEIZD; zs1=km)5>MQiv(l>gldpEbf9sN4Df4c%j72^vL;G;lX**ODu#s`nk%mAfK??8*3ufL zXKODAF{}=s6pP}5`~&1l+?yUR#_67 zn$feNQ8lEiS;O7aKQIu9DFa6ad*0t;hKdkIh>(`a;*9oChK4F8F?#ZRGL=q_rq$GE z5OlD1g>q7bR#GX|GT}|-ioe|Rs^O|gf&9ccdzyY;#}bv*L&bba58-@oMUcgH z=tz1u$OaOlnG=`zUmRWJysVx@WCv^YWI1xu!$QozCYulpx@6Dm*R|a#<@#Tj2BkzX!%X61=-j-RhAd1ewnx}sL zd^EA%)UhQ4t zKTX|D{qC*R1AQw8`f|JbZpCk({^jY1u~VxtWhJIO96P%@HoXFWo2MUo-YYZ^U-O5K zZ=b$G8aA#I-WkYy8!&U+di8eu&Gwvkd%k(&g8ivL8g@J@2*7{#w24iXoI6TWSA)A& zg1d6wU2DOX1@2>e-sit@;`)i%2)?4{YR^LKTHjjJraRI~)AnW0_Gbk*to}~_5yR9n_gh_&X5oH&DAFiAXq2G;V3Vu+pzvVZ5nuN{;g9Aim(O7 z_Vr%@`o4Mre2vs9E-R{x8X2e@!zw(vM&0nT-OJt$B|H~}>K#)wJ&XwLv}n-rC@T?Y z1nI8>s>u9|Xf#xqBQx`3I13#K^1Y*O-hI`*aQsHk^`2WQTI54V`NSk)#Nx6COrUTijjt*bi$$;1ESD3gwdG^ z+6_G&AB^tKYGXQp%t~L!-A>@Vz?!0p>*R*r?SbjtE z?Azb2d7IY>c&z0O;0Ld+Z45p_zj;=00_VGROyNIkeU<#ib;K)t>O0~RzHsn}T@n!J zJHY<`)(FMeXsJWGKvP*W2){%h_+Uezs$NSUab#8+JKGMIR4ZiXM?04~Z|}dk|Bjq% z3jcS71S%CmW%Ih|e-%HWv1((#qgfV;!tbe0pmc{DDHm`H!U#({Tm(Y+zcMGD^*zdF4|M}UDp sSM*(g3IfM*UlZro!~=i-ApWN!&owN_OKndHbiQ#!IWF*oAZDBW2WfJT^8f$< literal 0 HcmV?d00001 diff --git a/client/py/hopper.py b/client/py/hopper.py new file mode 100644 index 0000000..49af35e --- /dev/null +++ b/client/py/hopper.py @@ -0,0 +1,90 @@ +import os +import fcntl +import errno + +from enum import Enum + +class HopperPipeType(str, Enum): + IN = "in" + OUT = "out" + +class HopperPipe: + def __init__(self, type: HopperPipeType, name: str, endpoint: str, + hopper: str = "", nonblock: bool = False): + self.type = type + self.name = name + self.endpoint = endpoint + self.hopper = hopper + self.nonblock = nonblock + self._fd = -1 + + def _get_open_flags(self): + flags = 0 + if self.type == HopperPipeType.IN: + flags |= os.O_WRONLY + elif self.type == HopperPipeType.OUT: + flags |= os.O_RDONLY + if self.nonblock: + flags |= os.O_NONBLOCK + return flags + + def _get_path(self): + return os.path.join(self.hopper, self.endpoint, f"{self.name}.{str(self.type.value)}") + + def open(self): + if self.name == "" or self.endpoint == "": + raise ValueError("Name and endpoint must be set") + + if self.hopper == "": + hopper = os.getenv("HOPPER_PATH") + if hopper: + self.hopper = hopper + else: + raise ValueError("Hopper path not set, or HOPPER_PATH not available") + + path = self._get_path() + + try: + os.mkfifo(path, mode=0o660) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + open_flags = self._get_open_flags() + fd = os.open(path, open_flags) + + try: + fcntl.flock(fd, (fcntl.LOCK_EX if self.type == HopperPipeType.IN else fcntl.LOCK_SH) | fcntl.LOCK_NB); + except OSError as e: + if e.errno == errno.EWOULDBLOCK: + e.errno = EBUSY + errsv = e.errno + os.close(fd) + raise OSError(errno=errsv) + + self._fd = fd + + def close(self): + fcntl.flock(self._fd, fcntl.LOCK_UN) + os.close(self._fd) + + def read(self, len: int): + try: + buf = os.read(self._fd, len) + return buf + except OSError as e: + if e.errno == EWOULDBLOCK: + return b'' + else: + raise e + + def write(self, buf: bytes): + try: + res = os.write(self._fd, buf) + return res + except OSError as e: + if e.errno == EWOULDBLOCK: + return 0; + else: + raise e + diff --git a/client/py/hoppermodule.c b/client/py/hoppermodule.c deleted file mode 100644 index 08a1060..0000000 --- a/client/py/hoppermodule.c +++ /dev/null @@ -1,238 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include - -#include - -#include "hopper/hopper.h" -#include "hoppermodule.h" - -// define error type for hopper things -static PyObject *HopperError = NULL; - -// clang-format off -Hopper_Pipe_GET(name) -Hopper_Pipe_GET(endpoint) -Hopper_Pipe_GET(hopper) -Hopper_Pipe_SETSTR(name) -Hopper_Pipe_SETSTR(endpoint) -Hopper_Pipe_SETSTR(hopper); -// clang-format on - -static PyObject *hopper_pipe_new(PyTypeObject *type, PyObject *args, - PyObject *kwds) { - struct py_hopper_pipe *self = - (struct py_hopper_pipe *)type->tp_alloc(type, 0); - if (!self) - return NULL; - - // Create each field with default values - // This is __new__ in Python - self->name = PyUnicode_FromString(""); - self->endpoint = PyUnicode_FromString(""); - self->hopper = PyUnicode_FromString(""); - - if (!self->name || !self->endpoint || !self->hopper) { - Py_XDECREF(self->name); - Py_XDECREF(self->endpoint); - Py_XDECREF(self->hopper); - return NULL; - } - - self->fd = -1; - self->flags = 0; - - return (PyObject *)self; -} - -static int hopper_pipe_init(struct py_hopper_pipe *self, PyObject *args, - PyObject *kwds) { - static char *kwlist[] = {"name", "endpoint", "hopper", "fd", "flags", NULL}; - PyObject *name = NULL, *endpoint = NULL, *hopper = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUUii", kwlist, &name, - &endpoint, &hopper, &self->fd, - &self->flags)) - return -1; - - // After parsing, copy the provided value into the struct - if (name) - Py_SETREF(self->name, Py_NewRef(name)); - if (endpoint) - Py_SETREF(self->endpoint, Py_NewRef(endpoint)); - if (hopper) - Py_SETREF(self->hopper, Py_NewRef(hopper)); - - return 0; -} - -static void hopper_pipe_dealloc(struct py_hopper_pipe *self) { - assert(self->name); - assert(self->endpoint); - assert(self->hopper); - - Py_XDECREF(self->name); - Py_XDECREF(self->endpoint); - Py_XDECREF(self->hopper); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static PyObject *hopper_pipe_open(PyObject *self, PyObject *args) { - Hopper_Pipe_CONVERT(self, pipe); - - int res = hopper_open(&pipe); - if (res != 0) { - Hopper_Pipe_UPDATE(pipe, self); - return PyErr_SetFromErrno(HopperError); - } - - Hopper_Pipe_UPDATE(pipe, self); - Py_RETURN_NONE; -} - -static PyObject *hopper_pipe_close(PyObject *self, PyObject *args) { - Hopper_Pipe_CONVERT(self, pipe); - - int res = hopper_close(&pipe); - if (res != 0) { - Hopper_Pipe_UPDATE(pipe, self); - return PyErr_SetFromErrno(HopperError); - } - - Hopper_Pipe_UPDATE(pipe, self); - Py_RETURN_NONE; -} - -static PyObject *hopper_pipe_read(PyObject *self, PyObject *args) { - Py_ssize_t len; - if (!PyArg_ParseTuple(args, "n", &len)) - return NULL; - - PyObject *_dst = PyByteArray_FromStringAndSize(NULL, len); - if (!_dst) - return NULL; - - char *dst = PyByteArray_AS_STRING(_dst); - - Hopper_Pipe_CONVERT(self, pipe); - - ssize_t res = hopper_read(&pipe, (void *)dst, (size_t)len); - if (res < 0) { - Py_DECREF(_dst); - Hopper_Pipe_UPDATE(pipe, self); - return PyErr_SetFromErrno(HopperError); - } - - if (res < len) { - if (PyByteArray_Resize(_dst, (Py_ssize_t)res) < 0) { - Py_DECREF(_dst); - Hopper_Pipe_UPDATE(pipe, self); - return NULL; - } - } - - Hopper_Pipe_UPDATE(pipe, self); - return _dst; -} - -static PyObject *hopper_pipe_write(PyObject *self, PyObject *args) { - PyObject *_src = NULL; - if (!PyArg_ParseTuple(args, "O!", &PyBytes_Type, &_src)) - return NULL; - - char *src; - Py_ssize_t len; - if (PyBytes_AsStringAndSize(_src, &src, &len) != 0) - return NULL; - - Hopper_Pipe_CONVERT(self, pipe); - - ssize_t res = hopper_write(&pipe, (void *)src, (size_t)len); - if (res < 0) { - Hopper_Pipe_UPDATE(pipe, self); - return PyErr_SetFromErrno(HopperError); - } - - Hopper_Pipe_UPDATE(pipe, self); - return PyLong_FromSsize_t((Py_ssize_t)res); -} - -static PyMemberDef hopper_pipe_members[] = { - {"fd", Py_T_INT, offsetof(struct py_hopper_pipe, fd), 0, - "pipe file descriptor"}, - {"flags", Py_T_INT, offsetof(struct py_hopper_pipe, flags), 0, - "pipe flags"}, - {NULL}, -}; - -static PyGetSetDef hopper_pipe_get_set[] = { - {"name", (getter)_hopper_pipe_getname, (setter)_hopper_pipe_setname, - "pipe name", NULL}, - {"endpoint", (getter)_hopper_pipe_getendpoint, - (setter)_hopper_pipe_setendpoint, "pipe endpoint", NULL}, - {"hopper", (getter)_hopper_pipe_gethopper, (setter)_hopper_pipe_sethopper, - "pipe hopper", NULL}, - {NULL}, -}; - -static PyMethodDef hopper_pipe_methods[] = { - {"open", hopper_pipe_open, METH_VARARGS, "open a Hopper pipe"}, - {"close", hopper_pipe_close, METH_VARARGS, "close a Hopper pipe"}, - {"read", hopper_pipe_read, METH_VARARGS, "read bytes from a Hopper pipe"}, - {"write", hopper_pipe_write, METH_VARARGS, "write bytes to a Hopper pipe"}, - {NULL, NULL, 0, NULL}, -}; - -// clang-format off -static PyTypeObject hopper_pipe_type = { - .ob_base = PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "hopper.HopperPipe", - .tp_doc = PyDoc_STR("A object representing a Hopper pipe"), - .tp_basicsize = sizeof(struct py_hopper_pipe), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = hopper_pipe_new, - .tp_init = (initproc)hopper_pipe_init, - .tp_dealloc = (destructor)hopper_pipe_dealloc, - .tp_members = hopper_pipe_members, - .tp_getset = hopper_pipe_get_set, - .tp_methods = hopper_pipe_methods, -}; -// clang-format on - -static int hopper_module_exec(PyObject *m) { - if (HopperError != NULL) { - PyErr_SetString(PyExc_ImportError, - "cannot initialize hopper module multiple times"); - return -1; - } - - HopperError = PyErr_NewException("hopper.error", NULL, NULL); - - if (PyModule_AddObjectRef(m, "HopperError", HopperError) < 0) - return -1; - - if (PyType_Ready(&hopper_pipe_type) < 0) - return -1; - - if (PyModule_AddObjectRef(m, "HopperPipe", (PyObject *)&hopper_pipe_type) < - 0) - return -1; - - return 0; -} - -static PyModuleDef_Slot hopper_slots[] = { - {Py_mod_exec, hopper_module_exec}, - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - {0, NULL}, -}; - -static struct PyModuleDef hoppermodule = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "hopper", - .m_doc = NULL, - .m_slots = hopper_slots, -}; - -PyMODINIT_FUNC PyInit_hopper(void) { return PyModuleDef_Init(&hoppermodule); } - diff --git a/client/py/meson.build b/client/py/meson.build deleted file mode 100644 index 962f58b..0000000 --- a/client/py/meson.build +++ /dev/null @@ -1,13 +0,0 @@ -client_inc = include_directories('.', '../../include', '../../include/hopper/client/py') - -py = import('python').find_installation('python3') - -py.extension_module( - 'hopper', - 'hoppermodule.c', - c_args: ['-Wno-unused-parameter', '-Wno-pedantic'], - include_directories: client_inc, - link_with: libhopper, - install: true, -) - diff --git a/include/hopper/client/py/hoppermodule.h b/include/hopper/client/py/hoppermodule.h deleted file mode 100644 index 740ab96..0000000 --- a/include/hopper/client/py/hoppermodule.h +++ /dev/null @@ -1,63 +0,0 @@ -#ifndef hoppermodule_h_INCLUDED -#define hoppermodule_h_INCLUDED - -#define PY_SSIZE_T_CLEAN -#include - -// clang-format off -struct py_hopper_pipe { - PyObject_HEAD - PyObject *name; - PyObject *endpoint; - PyObject *hopper; - int fd; - int flags; -}; -// clang-format on - -// These macros are horrible, but hard to replace :( -#define Hopper_Pipe_GET(field) \ - static PyObject *_hopper_pipe_get##field(struct py_hopper_pipe *self, \ - void *closure) { \ - return Py_NewRef(self->field); \ - } - -#define Hopper_Pipe_SETSTR(field) \ - static int _hopper_pipe_set##field(struct py_hopper_pipe *self, \ - PyObject *value, void *closure) { \ - if (!value) { \ - PyErr_SetString(PyExc_TypeError, "cannot delete " #field); \ - return -1; \ - } \ - \ - if (!PyUnicode_Check(value)) { \ - PyErr_SetString(PyExc_TypeError, \ - #field " must be a Unicode string"); \ - return -1; \ - } \ - \ - Py_SETREF(self->field, Py_NewRef(value)); \ - return 0; \ - } - -#define Hopper_Pipe_CONVERT(in, out) \ - struct py_hopper_pipe *_self = (struct py_hopper_pipe *)in; \ - \ - const char *name = PyUnicode_AsUTF8(_self->name); \ - const char *endpoint = PyUnicode_AsUTF8(_self->endpoint); \ - const char *hopper = PyUnicode_AsUTF8(_self->hopper); \ - \ - struct hopper_pipe out = { \ - .name = name, \ - .endpoint = endpoint, \ - .hopper = hopper, \ - .fd = _self->fd, \ - .flags = _self->flags, \ - }; - -#define Hopper_Pipe_UPDATE(in, out) \ - struct py_hopper_pipe *_out = (struct py_hopper_pipe *)out; \ - _out->fd = in.fd; \ - _out->flags = in.flags; - -#endif // hoppermodule_h_INCLUDED From 8d296a5d7b4b5980e59169e295331c3c68675a05 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Tue, 20 Jan 2026 19:02:12 +0000 Subject: [PATCH 41/46] build: always build static when cross compiling --- .github/workflows/build_nix.yml | 4 ++-- flake.nix | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_nix.yml b/.github/workflows/build_nix.yml index db55b63..2a45c1f 100644 --- a/.github/workflows/build_nix.yml +++ b/.github/workflows/build_nix.yml @@ -17,7 +17,7 @@ jobs: uses: cachix/install-nix-action@v26 - name: Build Hopper x86_64 - run: nix build .#hopper.cross.x86_64 + run: nix build .#hopper.cross-x86_64-linux build-aarch64: runs-on: ubuntu-latest @@ -29,5 +29,5 @@ jobs: uses: cachix/install-nix-action@v26 - name: Build Hopper aarch64 - run: nix build .#hopper.cross.aarch64 + run: nix build .#hopper.cross-aarch64-linux diff --git a/flake.nix b/flake.nix index d8b5b98..a8bf585 100644 --- a/flake.nix +++ b/flake.nix @@ -27,13 +27,8 @@ packages = { hopper = { default = pkgs.callPackage ./nix/package.nix { }; - - cross = { - x86_64 = pkgs.pkgsCross.gnu64.callPackage ./nix/package.nix { }; - x86_64-static = pkgs.pkgsCross.gnu64.pkgsStatic.callPackage ./nix/package.nix { }; - aarch64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage ./nix/package.nix { }; - aarch64-static = pkgs.pkgsCross.aarch64-multiplatform.pkgsStatic.callPackage ./nix/package.nix { }; - }; + cross-x86_64-linux = pkgs.pkgsCross.gnu64.pkgsStatic.callPackage ./nix/package.nix { }; + cross-aarch64-linux = pkgs.pkgsCross.aarch64-multiplatform.pkgsStatic.callPackage ./nix/package.nix { }; }; }; From 4b3758f5544680cbecb5916d503747a2b2adb352 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 24 Jan 2026 10:35:38 +0000 Subject: [PATCH 42/46] client(lib,py): create endpoint directory when opening - create the endpoint directory when trying to open a pipe - return error code if directory creation failed, and it did not exist --- client/lib/lib.c | 14 ++++++++++++++ client/py/hopper.py | 8 +++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/client/lib/lib.c b/client/lib/lib.c index 9532b58..bd5bdf5 100644 --- a/client/lib/lib.c +++ b/client/lib/lib.c @@ -22,6 +22,12 @@ static int get_open_flags(int flags) { return open_flags; } +static char *get_endpoint_path(struct hopper_pipe *pipe) { + char *path = (char *)malloc(sizeof(char) * PATH_MAX); + sprintf(path, "%s/%s", pipe->hopper, pipe->endpoint); + return path; +} + static char *get_pipe_path(struct hopper_pipe *pipe) { char *path = (char *)malloc(sizeof(char) * PATH_MAX); const char *suffix = (pipe->flags & HOPPER_IN ? "in" : "out"); @@ -48,6 +54,14 @@ int hopper_open(struct hopper_pipe *pipe) { return -1; } + char *endpoint_path = get_endpoint_path(pipe); + res = mkdir(endpoint_path, 0755); + free(endpoint_path); + if (res == -1 && errno != EEXIST) { + pipe->fd = -1; + return -1; + } + char *pipe_path = get_pipe_path(pipe); if (mkfifo(pipe_path, 0660) < 0 && errno != EEXIST) { // mkfifo failed in some way, preserve errno diff --git a/client/py/hopper.py b/client/py/hopper.py index 49af35e..c4e5f59 100644 --- a/client/py/hopper.py +++ b/client/py/hopper.py @@ -28,8 +28,11 @@ def _get_open_flags(self): flags |= os.O_NONBLOCK return flags + def _get_endpoint_path(self): + return os.path.join(self.hopper, self.endpoint) + def _get_path(self): - return os.path.join(self.hopper, self.endpoint, f"{self.name}.{str(self.type.value)}") + return os.path.join(self._get_endpoint_path(), f"{self.name}.{str(self.type.value)}") def open(self): if self.name == "" or self.endpoint == "": @@ -42,6 +45,9 @@ def open(self): else: raise ValueError("Hopper path not set, or HOPPER_PATH not available") + endpoint_path = self._get_endpoint_path() + os.makedirs(endpoint_path, exist_ok=True) + path = self._get_path() try: From 1fb2fb9ad07587b8456164b612a4b3910a647421 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 24 Jan 2026 11:46:32 +0000 Subject: [PATCH 43/46] add `setup.py` file --- .gitignore | 11 +++++++++++ setup.py | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 5f20868..d24df54 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,14 @@ build/ compile_commands.json result + +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# Virtual environments +.venv diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..065b78e --- /dev/null +++ b/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup + +setup( + name="hopper", + version="0.1.0", + packages=["client.py"], + package_dir={"hopper": "client.py"}, + author="Nathan Gill", + author_email="nathan.j.gill@outlook.com", +) From 27d01d1bd1fff9fcb9ff7fa1cd913e9705f57a35 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 24 Jan 2026 11:51:49 +0000 Subject: [PATCH 44/46] use correct path in `setup.py` --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 065b78e..5abade5 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ setup( name="hopper", version="0.1.0", - packages=["client.py"], - package_dir={"hopper": "client.py"}, + packages=["hopper"], + package_dir={"hopper": "client/py"}, author="Nathan Gill", author_email="nathan.j.gill@outlook.com", ) From 9bd5f234137456822b7e644840705ee4d39e71d1 Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 24 Jan 2026 12:21:24 +0000 Subject: [PATCH 45/46] daemon: open any existing pipes when creating an endpoint --- daemon/daemon_endpoint.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/daemon/daemon_endpoint.cpp b/daemon/daemon_endpoint.cpp index dbd6386..54dd85a 100644 --- a/daemon/daemon_endpoint.cpp +++ b/daemon/daemon_endpoint.cpp @@ -1,4 +1,6 @@ #include "hopper/daemon/daemon.hpp" +#include "hopper/daemon/util.hpp" +#include #include namespace hopper { @@ -19,6 +21,22 @@ uint32_t HopperDaemon::create_endpoint(const std::filesystem::path &path) { std::cout << "CREATE " << *endpoint << std::endl; + // Open anything that may already exist in the endpoint + for (const auto &dir_entry : std::filesystem::directory_iterator{path}) { + const auto &p = dir_entry.path(); + + PipeType pipe_type = detect_pipe_type(p); + if (pipe_type == PipeType::NONE) + continue; + + HopperPipe *pipe = + (pipe_type == PipeType::IN ? endpoint->add_input_pipe(p) + : endpoint->add_output_pipe(p)); + + if (pipe != nullptr) + add_pipe(pipe); + } + return endpoint_id; } From c124255e303079bd2de49ba63bb6dfbfc9a18c9a Mon Sep 17 00:00:00 2001 From: Nathan Gill Date: Sat, 24 Jan 2026 13:36:13 +0000 Subject: [PATCH 46/46] client(py): rename `_fd` to `fd` --- client/py/hopper.py | 13 +++++++------ daemon/daemon_endpoint.cpp | 3 +-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/py/hopper.py b/client/py/hopper.py index c4e5f59..8ec85f2 100644 --- a/client/py/hopper.py +++ b/client/py/hopper.py @@ -16,7 +16,7 @@ def __init__(self, type: HopperPipeType, name: str, endpoint: str, self.endpoint = endpoint self.hopper = hopper self.nonblock = nonblock - self._fd = -1 + self.fd = -1 def _get_open_flags(self): flags = 0 @@ -68,15 +68,16 @@ def open(self): os.close(fd) raise OSError(errno=errsv) - self._fd = fd + self.fd = fd def close(self): - fcntl.flock(self._fd, fcntl.LOCK_UN) - os.close(self._fd) + fcntl.flock(self.fd, fcntl.LOCK_UN) + os.close(self.fd) + self.fd = -1 def read(self, len: int): try: - buf = os.read(self._fd, len) + buf = os.read(self.fd, len) return buf except OSError as e: if e.errno == EWOULDBLOCK: @@ -86,7 +87,7 @@ def read(self, len: int): def write(self, buf: bytes): try: - res = os.write(self._fd, buf) + res = os.write(self.fd, buf) return res except OSError as e: if e.errno == EWOULDBLOCK: diff --git a/daemon/daemon_endpoint.cpp b/daemon/daemon_endpoint.cpp index 54dd85a..5c368de 100644 --- a/daemon/daemon_endpoint.cpp +++ b/daemon/daemon_endpoint.cpp @@ -12,9 +12,8 @@ uint32_t HopperDaemon::create_endpoint(const std::filesystem::path &path) { int inotify_watch_fd = inotify_add_watch(m_inotify_fd, path.c_str(), IN_CREATE | IN_DELETE); - if (inotify_watch_fd < 0) { + if (inotify_watch_fd < 0) return 0; - } auto *endpoint = new HopperEndpoint(endpoint_id, inotify_watch_fd, path); m_endpoints[endpoint_id] = endpoint;