From 303184b5444862a4aece50d5db022a71d1d09a00 Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Sat, 27 Jan 2024 15:18:21 +0200 Subject: [PATCH 1/3] nip-114: filter.ids_only --- src/apps/relay/RelayReqMonitor.cpp | 7 ++++++- src/apps/relay/RelayReqWorker.cpp | 12 +++++++++--- src/apps/relay/RelayServer.h | 15 +++++++++++++++ src/apps/relay/RelayWebsocket.cpp | 2 +- src/filters.h | 11 +++++++++++ 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/apps/relay/RelayReqMonitor.cpp b/src/apps/relay/RelayReqMonitor.cpp index 4fadd701..0c88c96e 100644 --- a/src/apps/relay/RelayReqMonitor.cpp +++ b/src/apps/relay/RelayReqMonitor.cpp @@ -32,7 +32,12 @@ void RelayServer::runReqMonitor(ThreadPool::Thread &thr) { env.foreach_Event(txn, [&](auto &ev){ if (msg->sub.filterGroup.doesMatch(ev.flat_nested())) { - sendEvent(connId, msg->sub.subId, getEventJson(txn, decomp, ev.primaryKeyId)); + if (msg->sub.filterGroup.ids_only()) { + auto id = to_hex(sv(ev.flat_nested()->id())); + sendHave(connId, msg->sub.subId, id); + } else { + sendEvent(connId, msg->sub.subId, getEventJson(txn, decomp, ev.primaryKeyId)); + } } return true; diff --git a/src/apps/relay/RelayReqWorker.cpp b/src/apps/relay/RelayReqWorker.cpp index 92ec2f31..6556abcc 100644 --- a/src/apps/relay/RelayReqWorker.cpp +++ b/src/apps/relay/RelayReqWorker.cpp @@ -6,11 +6,17 @@ void RelayServer::runReqWorker(ThreadPool::Thread &thr) { Decompressor decomp; QueryScheduler queries; - queries.onEvent = [&](lmdb::txn &txn, const auto &sub, uint64_t levId, std::string_view eventPayload){ - sendEvent(sub.connId, sub.subId, decodeEventPayload(txn, decomp, eventPayload, nullptr, nullptr)); + queries.onEvent = [&](lmdb::txn &txn, const auto &sub, uint64_t levId, std::string_view eventPayload) { + if (sub.filterGroup.ids_only()) { + auto ev = lookupEventByLevId(txn, levId); + auto id = to_hex(sv(ev.flat_nested()->id())); + sendHave(sub.connId, sub.subId, id); + } else { + sendEvent(sub.connId, sub.subId, decodeEventPayload(txn, decomp, eventPayload, nullptr, nullptr)); + } }; - queries.onComplete = [&](lmdb::txn &, Subscription &sub){ + queries.onComplete = [&](lmdb::txn &, Subscription &sub) { sendToConn(sub.connId, tao::json::to_string(tao::json::value::array({ "EOSE", sub.subId.str() }))); tpReqMonitor.dispatch(sub.connId, MsgReqMonitor{MsgReqMonitor::NewSub{std::move(sub)}}); }; diff --git a/src/apps/relay/RelayServer.h b/src/apps/relay/RelayServer.h index 407e09ff..5212f4f9 100644 --- a/src/apps/relay/RelayServer.h +++ b/src/apps/relay/RelayServer.h @@ -211,6 +211,21 @@ struct RelayServer { sendToConn(connId, std::move(reply)); } + void sendHave(uint64_t connId, const SubId &subId, const std::string_view eventId) { + auto subIdSv = subId.sv(); + + std::string reply; + reply.reserve(14 + subIdSv.size() + eventId.size()); + + reply += "[\"HAVE\",\""; + reply += subIdSv; + reply += "\",\""; + reply += eventId; + reply += "\"]"; + + sendToConn(connId, std::move(reply)); + } + void sendEventToBatch(RecipientList &&list, std::string &&evJson) { tpWebsocket.dispatch(0, MsgWebsocket{MsgWebsocket::SendEventToBatch{std::move(list), std::move(evJson)}}); hubTrigger->send(); diff --git a/src/apps/relay/RelayWebsocket.cpp b/src/apps/relay/RelayWebsocket.cpp index ed115532..a045f77d 100644 --- a/src/apps/relay/RelayWebsocket.cpp +++ b/src/apps/relay/RelayWebsocket.cpp @@ -48,7 +48,7 @@ void RelayServer::runWebsocket(ThreadPool::Thread &thr) { tempBuf.reserve(cfg().events__maxEventSize + MAX_SUBID_SIZE + 100); - tao::json::value supportedNips = tao::json::value::array({ 1, 2, 4, 9, 11, 12, 16, 20, 22, 28, 33, 40 }); + tao::json::value supportedNips = tao::json::value::array({ 1, 2, 4, 9, 11, 12, 16, 20, 22, 28, 33, 40, 114 }); auto getServerInfoHttpResponse = [&supportedNips, ver = uint64_t(0), rendered = std::string("")]() mutable { if (ver != cfg().version()) { diff --git a/src/filters.h b/src/filters.h index 22a82d32..8fe9868a 100644 --- a/src/filters.h +++ b/src/filters.h @@ -116,6 +116,7 @@ struct NostrFilter { uint64_t limit = MAX_U64; bool neverMatch = false; bool indexOnlyScans = false; + bool idsOnly = false; explicit NostrFilter(const tao::json::value &filterObj, uint64_t maxFilterLimit) { uint64_t numMajorFields = 0; @@ -154,6 +155,8 @@ struct NostrFilter { until = v.get_unsigned(); } else if (k == "limit") { limit = v.get_unsigned(); + } else if (k == "ids_only") { + idsOnly = v.get_boolean(); } else { throw herr("unrecognised filter item"); } @@ -248,6 +251,14 @@ struct NostrFilterGroup { return false; } + bool ids_only() const { + for (const auto &f : filters) { + if (f.idsOnly) return true; + } + + return false; + } + size_t size() const { return filters.size(); } From 77803bae550afed6cdc51cac3eb824ddb67daa9d Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Wed, 31 Jan 2024 12:23:28 +0200 Subject: [PATCH 2/3] README.md: sync --filter --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9790e48b..81fa5ee6 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ This will download all missing events from the remote relay and insert them into Instead of a "full DB" sync, you can also sync the result of a nostr filter (or multiple filters, use a JSON array of them): - ./strfry sync wss://relay.example.com '{"authors":["003b"]}' + ./strfry sync wss://relay.example.com --filter '{"authors":["003b"]}' Warning: Syncing can consume a lot of memory and bandwidth if the DBs are highly divergent (for example if your local DB is empty and your filter matches many events). From 8c4064e497fc41338f3b8ca8f86cd6f94b3eff6a Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Wed, 31 Jan 2024 13:32:28 +0200 Subject: [PATCH 3/3] add GET command handling --- src/apps/relay/RelayIngester.cpp | 20 ++++++++++++++++++++ src/apps/relay/RelayServer.h | 1 + 2 files changed, 21 insertions(+) diff --git a/src/apps/relay/RelayIngester.cpp b/src/apps/relay/RelayIngester.cpp index d553f5d2..063409a3 100644 --- a/src/apps/relay/RelayIngester.cpp +++ b/src/apps/relay/RelayIngester.cpp @@ -43,6 +43,14 @@ void RelayServer::runIngester(ThreadPool::Thread &thr) { } catch (std::exception &e) { sendNoticeError(msg->connId, std::string("bad req: ") + e.what()); } + } else if (cmd == "GET") { + if (cfg().relay__logging__dumpInReqs) LI << "[" << msg->connId << "] dumpInReq: " << msg->payload; + + try { + ingesterProcessGet(txn, decomp, msg->connId, arr); + } catch (std::exception &e) { + sendNoticeError(msg->connId, std::string("bad get: ") + e.what()); + } } else if (cmd == "CLOSE") { if (cfg().relay__logging__dumpInReqs) LI << "[" << msg->connId << "] dumpInReq: " << msg->payload; @@ -113,6 +121,18 @@ void RelayServer::ingesterProcessReq(lmdb::txn &txn, uint64_t connId, const tao: tpReqWorker.dispatch(connId, MsgReqWorker{MsgReqWorker::NewSub{std::move(sub)}}); } +void RelayServer::ingesterProcessGet(lmdb::txn &txn, Decompressor &decomp, uint64_t connId, const tao::json::value &arr) { + if (arr.get_array().size() != 2) throw herr("GET arr size != 2"); + + auto ev = lookupEventById(txn, from_hex(arr[1].get_string())); + if (!ev) { + sendNoticeError(connId, std::string("GET event not found")); + } else { + auto evJson = getEventJson(txn, decomp, ev->primaryKeyId); + sendEvent(connId, SubId("*"), evJson); + } +} + void RelayServer::ingesterProcessClose(lmdb::txn &txn, uint64_t connId, const tao::json::value &arr) { if (arr.get_array().size() != 2) throw herr("arr too small/big"); diff --git a/src/apps/relay/RelayServer.h b/src/apps/relay/RelayServer.h index 5212f4f9..33f09549 100644 --- a/src/apps/relay/RelayServer.h +++ b/src/apps/relay/RelayServer.h @@ -169,6 +169,7 @@ struct RelayServer { void runIngester(ThreadPool::Thread &thr); void ingesterProcessEvent(lmdb::txn &txn, uint64_t connId, std::string ipAddr, secp256k1_context *secpCtx, const tao::json::value &origJson, std::vector &output); void ingesterProcessReq(lmdb::txn &txn, uint64_t connId, const tao::json::value &origJson); + void ingesterProcessGet(lmdb::txn &txn, Decompressor &decomp, uint64_t connId, const tao::json::value &origJson); void ingesterProcessClose(lmdb::txn &txn, uint64_t connId, const tao::json::value &origJson); void ingesterProcessNegentropy(lmdb::txn &txn, Decompressor &decomp, uint64_t connId, const tao::json::value &origJson);