From 053e3a37421b154f8bc372cff457dd3d20e86c0e Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Wed, 11 Nov 2026 15:25:59 -0500 Subject: [PATCH 1/7] connector: take the socket setup and framing of the POSIX link out of posix_connector The endpoint parsing, the creation and tuning of a socket, listen, connect and admit, the TCP length prefix, the SCTP send info and the test for an SCTP notification now live in socket_link.{hpp,cpp}, so a second link can put the same bytes on the same sockets. posix_connector.cpp keeps what is its own: the threads, the queues, the per-peer state. Nothing a peer can observe changes and no POSIX test changed. test_socket_link pins the framing the two links must share: a whole PDU, a partial one, one of the limit and one a byte over it, the prefix written, and the Invalid-PDU-length message the POSIX link logs. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- cmake/libe3Sources.cmake | 1 + cmake/libe3Tests.cmake | 5 + src/connector/posix_connector.cpp | 424 +++------------------------- src/connector/posix_connector.hpp | 21 +- src/connector/socket_link.cpp | 446 ++++++++++++++++++++++++++++++ src/connector/socket_link.hpp | 198 +++++++++++++ tests/test_socket_link.cpp | 120 ++++++++ 7 files changed, 808 insertions(+), 407 deletions(-) create mode 100644 src/connector/socket_link.cpp create mode 100644 src/connector/socket_link.hpp create mode 100644 tests/test_socket_link.cpp diff --git a/cmake/libe3Sources.cmake b/cmake/libe3Sources.cmake index ed177fe8..f35e4b9e 100644 --- a/cmake/libe3Sources.cmake +++ b/cmake/libe3Sources.cmake @@ -40,6 +40,7 @@ set(LIBE3_SOURCES # Connector src/connector/connector_factory.cpp src/connector/posix_connector.cpp + src/connector/socket_link.cpp src/c_api.cpp ) diff --git a/cmake/libe3Tests.cmake b/cmake/libe3Tests.cmake index 6bab0e94..bc27283e 100644 --- a/cmake/libe3Tests.cmake +++ b/cmake/libe3Tests.cmake @@ -137,6 +137,11 @@ foreach(test_src IN LISTS LIBE3_TEST_SOURCES) LIBE3_FIXTURE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures") endif() + if(simple_name STREQUAL "socket_link") + # It reaches into the connectors' own header: the framing. + target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + endif() + add_test(NAME ${target_name} COMMAND ${target_name}) endforeach() diff --git a/src/connector/posix_connector.cpp b/src/connector/posix_connector.cpp index c40baf6c..4ecd01bf 100644 --- a/src/connector/posix_connector.cpp +++ b/src/connector/posix_connector.cpp @@ -10,16 +10,12 @@ #include "libe3/logger.hpp" #include #include -#include #include -#include #include -#include #ifdef LIBE3_ENABLE_SCTP #include #endif #include -#include #include #include #include @@ -31,22 +27,11 @@ namespace libe3 { +using namespace socket_link; + namespace { constexpr const char* LOG_TAG = "PosixConn"; -constexpr const char* IPC_BASE_DIR = "/tmp/dapps"; -// What E3Config holds for setup_endpoint unless the program sets it. The ZeroMQ link -// shares the string, so the POSIX link maps it to its own default instead of changing it. -constexpr const char* SHARED_DEFAULT_ENDPOINT = "ipc:///tmp/dapps/setup"; -constexpr const char* DEFAULT_IPC_PATH = "/tmp/dapps/e3"; constexpr int POLL_TIMEOUT_MS = 500; // How long one read waits, so shutdown is noticed -constexpr int SEND_TIMEOUT_MS = 500; // Per-peer send timeout (drop a stalled peer) -constexpr int CONNECT_ATTEMPTS = 10; // The dApp waits for a RAN that is not up yet -constexpr int CONNECT_PAUSE_MS = 200; -constexpr int LISTEN_BACKLOG = 64; -constexpr size_t MAX_PDU = MAX_PROTOCOL_DATA_SIZE * 2; // One limit for every transport and direction -constexpr uint32_t SCTP_PPID = 0x00E3A001; // Project-private; no payload protocol id is registered for E3AP -constexpr int SCTP_PROCEDURE_STREAM = 0; -constexpr int SCTP_INDICATION_STREAM = 1; constexpr int64_t SETUP_PUMP_AFTER_NS = 1'000'000'000; // The setup thread reads if receive() has been idle this long int64_t now_ns() { @@ -60,51 +45,6 @@ int wait_readable(int fd, int timeout_ms) { return poll(&pfd, 1, timeout_ms); } -/** - * @brief Strip a URI scheme ("ipc://", "tcp://") from an endpoint, leaving a path. - */ -std::string strip_scheme(const std::string& endpoint) { - auto pos = endpoint.find("://"); - return pos == std::string::npos ? endpoint : endpoint.substr(pos + 3); -} - -/** - * @brief Create a socket of the SCTP protocol. - * - * SCTP is optional at build time (LIBE3_ENABLE_SCTP, off by default). The enumerator - * stays in the ABI either way, so asking for SCTP against a build without it has to - * fail loudly here rather than fall through to TCP, which would look as if it worked - * while putting traffic on the wrong protocol. Returns -1 with errno set, like socket(). - */ -int sctp_socket(int family) { -#ifdef LIBE3_ENABLE_SCTP - return socket(family, SOCK_STREAM, IPPROTO_SCTP); -#else - (void)family; - E3_LOG_ERROR(LOG_TAG) << "SCTP transport requested but libe3 was built without it. " - "Rebuild with -DLIBE3_ENABLE_SCTP=ON (needs libsctp-dev), " - "or use the TCP or IPC transport."; - errno = EPROTONOSUPPORT; - return -1; -#endif -} - -void set_timeout(int fd, int option, int timeout_ms, const char* name) { - struct timeval tv; - tv.tv_sec = timeout_ms / 1000; - tv.tv_usec = (timeout_ms % 1000) * 1000; - if (setsockopt(fd, SOL_SOCKET, option, &tv, sizeof(tv)) != 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to set " << name << ": " << strerror(errno); - } -} - -void set_nonblocking(int fd) { - int flags = fcntl(fd, F_GETFL, 0); - if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to set O_NONBLOCK: " << strerror(errno); - } -} - // --------------------------------------------------------------------------- // One PDU per message // --------------------------------------------------------------------------- @@ -118,21 +58,20 @@ enum class Read { PDU, NONE, CLOSED, ERROR }; * The prefix is four bytes in network order; zero and more than MAX_PDU are refused. */ Read pop_stream_frame(std::vector& rx, std::vector& out) { - if (rx.size() < sizeof(uint32_t)) { - return Read::NONE; - } - uint32_t size = 0; - std::memcpy(&size, rx.data(), sizeof(size)); - size = ntohl(size); - if (size == 0 || size > MAX_PDU) { - E3_LOG_ERROR(LOG_TAG) << "Invalid PDU length " << size; - return Read::ERROR; - } - if (rx.size() < sizeof(uint32_t) + size) { - return Read::NONE; + size_t size = 0; + switch (parse_stream_frame(rx.data(), rx.size(), size)) { + case StreamFrame::NEED_MORE: + return Read::NONE; + case StreamFrame::INVALID: + E3_LOG_ERROR(LOG_TAG) << "Invalid PDU length " << size; + return Read::ERROR; + case StreamFrame::PDU: + break; } - out.assign(rx.begin() + sizeof(uint32_t), rx.begin() + sizeof(uint32_t) + size); - rx.erase(rx.begin(), rx.begin() + sizeof(uint32_t) + size); + const auto start = static_cast(TCP_PREFIX_BYTES); + const auto end = start + static_cast(size); + out.assign(rx.begin() + start, rx.begin() + end); + rx.erase(rx.begin(), rx.begin() + end); return Read::PDU; } @@ -206,18 +145,8 @@ Read read_one(int fd, E3TransportLayer transport, std::vector& rx, std: if (msg.msg_flags & MSG_NOTIFICATION) { return Read::NONE; } - if (rx.empty()) { - for (struct cmsghdr* c = CMSG_FIRSTHDR(&msg); c != nullptr; c = CMSG_NXTHDR(&msg, c)) { - if (c->cmsg_level == IPPROTO_SCTP && c->cmsg_type == SCTP_RCVINFO) { - struct sctp_rcvinfo info; - std::memcpy(&info, CMSG_DATA(c), sizeof(info)); - const uint32_t ppid = ntohl(info.rcv_ppid); - if (ppid != 0 && ppid != SCTP_PPID) { - E3_LOG_ERROR(LOG_TAG) << "Message with payload protocol id 0x" << std::hex << ppid; - return Read::ERROR; - } - } - } + if (rx.empty() && !sctp_ppid_acceptable(msg, LOG_TAG)) { + return Read::ERROR; } rx.insert(rx.end(), chunk.begin(), chunk.begin() + n); if (rx.size() > MAX_PDU) { @@ -251,9 +180,10 @@ ErrorCode write_pdu(int fd, E3TransportLayer transport, const std::vector(data.size())); + uint8_t prefix[TCP_PREFIX_BYTES]; + write_stream_prefix(prefix, data.size()); struct iovec iov[2]; - iov[0].iov_base = &prefix; + iov[0].iov_base = prefix; iov[0].iov_len = sizeof(prefix); iov[1].iov_base = const_cast(data.data()); iov[1].iov_len = data.size(); @@ -299,21 +229,12 @@ ErrorCode write_pdu(int fd, E3TransportLayer transport, const std::vector(data.data()), data.size()}; struct msghdr msg{}; msg.msg_iov = &iov; msg.msg_iovlen = 1; - msg.msg_control = control; - msg.msg_controllen = sizeof(control); - struct cmsghdr* c = CMSG_FIRSTHDR(&msg); - c->cmsg_level = IPPROTO_SCTP; - c->cmsg_type = SCTP_SNDINFO; - c->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndinfo)); - struct sctp_sndinfo info{}; - info.snd_sid = kind == PduClass::INDICATION ? SCTP_INDICATION_STREAM : SCTP_PROCEDURE_STREAM; - info.snd_ppid = htonl(SCTP_PPID); - std::memcpy(CMSG_DATA(c), &info, sizeof(info)); + set_sctp_send_info(msg, control, kind); ssize_t n; do { n = ::sendmsg(fd, &msg, MSG_NOSIGNAL); @@ -329,22 +250,6 @@ ErrorCode write_pdu(int fd, E3TransportLayer transport, const std::vector(&ss); - inet_ntop(AF_INET, &in->sin_addr, text, sizeof(text)); - } else if (ss.ss_family == AF_INET6) { - const auto* in6 = reinterpret_cast(&ss); - if (IN6_IS_ADDR_V4MAPPED(&in6->sin6_addr)) { - inet_ntop(AF_INET, &in6->sin6_addr.s6_addr[12], text, sizeof(text)); - } else { - inet_ntop(AF_INET6, &in6->sin6_addr, text, sizeof(text)); - } - } - return text; -} } // anonymous namespace /** @brief One accepted connection of a RAN. Closes its socket when the last reference goes. */ @@ -388,11 +293,7 @@ PosixE3Connector::PosixE3Connector( , port_(port) , options_(options) { - if (transport_layer == E3TransportLayer::IPC) { - setup_endpoint_ = endpoint == SHARED_DEFAULT_ENDPOINT ? DEFAULT_IPC_PATH : strip_scheme(endpoint); - } else { - setup_endpoint_ = endpoint; - } + setup_endpoint_ = link_endpoint(transport_layer, endpoint); E3_LOG_INFO(LOG_TAG) << "Creating POSIX connector, one endpoint: " << setup_endpoint_ << " port " << port_; } @@ -401,266 +302,8 @@ PosixE3Connector::~PosixE3Connector() { dispose(); } -PosixE3Connector::Target PosixE3Connector::target() const { - Target where{"", std::to_string(port_)}; - std::string rest; - for (const char* scheme : {"tcp://", "sctp://"}) { - if (endpoint_uri_.rfind(scheme, 0) == 0) { - rest = endpoint_uri_.substr(std::strlen(scheme)); - break; - } - } - if (rest.empty()) { - return where; - } - if (rest[0] == '[') { - const auto close = rest.find(']'); - if (close == std::string::npos) { - return where; - } - where.host = rest.substr(1, close - 1); - if (close + 1 < rest.size() && rest[close + 1] == ':') { - where.port = rest.substr(close + 2); - } - } else { - const auto colon = rest.rfind(':'); - if (colon != std::string::npos && rest.find(':') == colon) { - where.host = rest.substr(0, colon); - where.port = rest.substr(colon + 1); - } else { - where.host = rest; - } - } - if (where.port.empty()) { - where.port = std::to_string(port_); - } - return where; -} - -int PosixE3Connector::make_socket(int family) const { - switch (transport_layer_) { - case E3TransportLayer::IPC: - return socket(AF_UNIX, SOCK_SEQPACKET, 0); - case E3TransportLayer::SCTP: - return sctp_socket(family); - default: - return socket(family, SOCK_STREAM, 0); - } -} - -void PosixE3Connector::tune_socket(int fd, SocketKind kind) const { - if (transport_layer_ == E3TransportLayer::IPC) { - if (kind == SocketKind::ACCEPTED) { - set_timeout(fd, SO_SNDTIMEO, SEND_TIMEOUT_MS, "SO_SNDTIMEO"); - } - return; - } - const int one = 1; - if (kind == SocketKind::LISTENER) { - setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); - } - if (kind != SocketKind::LISTENER) { - set_timeout(fd, SO_SNDTIMEO, SEND_TIMEOUT_MS, "SO_SNDTIMEO"); - } - if (transport_layer_ == E3TransportLayer::TCP) { - if (kind != SocketKind::LISTENER && setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)) != 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to set TCP_NODELAY: " << strerror(errno); - } - return; - } -#ifdef LIBE3_ENABLE_SCTP - if (kind != SocketKind::ACCEPTED) { - // Before the association exists, on both peers: two streams, and buffers and - // delayed SACK that do not stall a message larger than the Linux defaults allow. - struct sctp_initmsg init{}; - init.sinit_num_ostreams = 2; - init.sinit_max_instreams = 2; - if (setsockopt(fd, IPPROTO_SCTP, SCTP_INITMSG, &init, sizeof(init)) != 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to request two SCTP streams: " << strerror(errno); - } - if (options_.socket_buffer_bytes > 0) { - const int size = static_cast(options_.socket_buffer_bytes); - setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)); - setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)); - } - if (!options_.sctp_delayed_sack) { - struct sctp_sack_info sack{}; - sack.sack_assoc_id = 0; - sack.sack_delay = 0; - sack.sack_freq = 1; - if (setsockopt(fd, IPPROTO_SCTP, SCTP_DELAYED_SACK, &sack, sizeof(sack)) != 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to turn delayed SACK off: " << strerror(errno); - } - } - } - if (kind != SocketKind::LISTENER) { - if (setsockopt(fd, IPPROTO_SCTP, SCTP_NODELAY, &one, sizeof(one)) != 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to set SCTP_NODELAY: " << strerror(errno); - } - if (setsockopt(fd, IPPROTO_SCTP, SCTP_RECVRCVINFO, &one, sizeof(one)) != 0) { - E3_LOG_WARN(LOG_TAG) << "Failed to ask for the SCTP receive info: " << strerror(errno); - } - } -#endif -} - -int PosixE3Connector::listen_on(const Target& where) { - int fd = -1; - int result = -1; - if (transport_layer_ == E3TransportLayer::IPC) { - struct stat st{}; - if (stat(IPC_BASE_DIR, &st) == -1 && mkdir(IPC_BASE_DIR, 0777) == -1 && errno != EEXIST) { - E3_LOG_ERROR(LOG_TAG) << "Failed to create IPC directory: " << strerror(errno); - return -1; - } - chmod(IPC_BASE_DIR, 0777); - if (setup_endpoint_.size() >= sizeof(sockaddr_un::sun_path)) { - E3_LOG_ERROR(LOG_TAG) << "IPC path is too long: " << setup_endpoint_; - return -1; - } - fd = make_socket(AF_UNIX); - if (fd < 0) { - return -1; - } - struct sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - std::memcpy(addr.sun_path, setup_endpoint_.c_str(), setup_endpoint_.size()); - unlink(setup_endpoint_.c_str()); - result = bind(fd, reinterpret_cast(&addr), sizeof(addr)); - } else if (where.host.empty()) { - // Any address, both families: a dual-stack socket, or IPv4 alone where there is no IPv6. - fd = make_socket(AF_INET6); - if (fd >= 0) { - const int off = 0; - setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &off, sizeof(off)); - tune_socket(fd, SocketKind::LISTENER); - struct sockaddr_in6 addr{}; - addr.sin6_family = AF_INET6; - addr.sin6_addr = in6addr_any; - addr.sin6_port = htons(static_cast(std::stoi(where.port))); - result = bind(fd, reinterpret_cast(&addr), sizeof(addr)); - if (result < 0) { - close(fd); - fd = -1; - } - } - if (fd < 0) { - fd = make_socket(AF_INET); - if (fd < 0) { - return -1; - } - tune_socket(fd, SocketKind::LISTENER); - struct sockaddr_in addr{}; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = htons(static_cast(std::stoi(where.port))); - result = bind(fd, reinterpret_cast(&addr), sizeof(addr)); - } - } else { - struct addrinfo hints{}; - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE; - struct addrinfo* found = nullptr; - if (getaddrinfo(where.host.c_str(), where.port.c_str(), &hints, &found) != 0 || found == nullptr) { - E3_LOG_ERROR(LOG_TAG) << "Cannot resolve " << where.host; - return -1; - } - fd = make_socket(found->ai_family); - if (fd >= 0) { - tune_socket(fd, SocketKind::LISTENER); - result = bind(fd, found->ai_addr, found->ai_addrlen); - } - freeaddrinfo(found); - if (fd < 0) { - return -1; - } - } - if (result < 0 || listen(fd, LISTEN_BACKLOG) < 0) { - E3_LOG_ERROR(LOG_TAG) << "Failed to listen on " << setup_endpoint_ << ": " << strerror(errno); - close(fd); - return -1; - } - set_nonblocking(fd); - return fd; -} - -int PosixE3Connector::connect_to(const Target& where) { - if (transport_layer_ == E3TransportLayer::IPC) { - if (setup_endpoint_.size() >= sizeof(sockaddr_un::sun_path)) { - errno = ENAMETOOLONG; - return -1; - } - int fd = make_socket(AF_UNIX); - if (fd < 0) { - return -1; - } - struct sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - std::memcpy(addr.sun_path, setup_endpoint_.c_str(), setup_endpoint_.size()); - if (connect(fd, reinterpret_cast(&addr), sizeof(addr)) < 0) { - const int saved = errno; - close(fd); - errno = saved; - return -1; - } - tune_socket(fd, SocketKind::CLIENT); - return fd; - } - struct addrinfo hints{}; - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - struct addrinfo* found = nullptr; - const std::string host = where.host.empty() ? "127.0.0.1" : where.host; - if (getaddrinfo(host.c_str(), where.port.c_str(), &hints, &found) != 0 || found == nullptr) { - errno = EHOSTUNREACH; - return -1; - } - int fd = -1; - int saved = ECONNREFUSED; - for (struct addrinfo* a = found; a != nullptr && fd < 0; a = a->ai_next) { - int candidate = make_socket(a->ai_family); - if (candidate < 0) { - saved = errno; - continue; - } - tune_socket(candidate, SocketKind::CLIENT); - if (connect(candidate, a->ai_addr, a->ai_addrlen) == 0) { - fd = candidate; - } else { - saved = errno; - close(candidate); - } - } - freeaddrinfo(found); - if (fd < 0) { - errno = saved; - } - return fd; -} - -bool PosixE3Connector::admit(const void* address) const { - if (transport_layer_ == E3TransportLayer::IPC) { - return true; - } - const auto& ss = *static_cast(address); - if (!options_.allowed_peers.empty()) { - const std::string text = address_text(ss); - if (std::find(options_.allowed_peers.begin(), options_.allowed_peers.end(), text) - == options_.allowed_peers.end()) { - E3_LOG_WARN(LOG_TAG) << "Refused a connection from " << text << ": not in the allowed peers"; - return false; - } - } - if (options_.maximum_associations > 0) { - std::lock_guard lock(peers_mutex_); - if (peers_.size() >= options_.maximum_associations) { - E3_LOG_WARN(LOG_TAG) << "Refused a connection from " << address_text(ss) - << ": " << options_.maximum_associations << " peers are connected"; - return false; - } - } - return true; +SocketSpec PosixE3Connector::spec() const { + return SocketSpec{transport_layer_, setup_endpoint_, parse_target(endpoint_uri_, port_), options_, LOG_TAG}; } // =========================================================================== @@ -668,7 +311,7 @@ bool PosixE3Connector::admit(const void* address) const { // =========================================================================== ErrorCode PosixE3Connector::setup_initial_connection() { - listener_ = listen_on(target()); + listener_ = listen_on(spec(), /*nonblocking=*/true); if (listener_ < 0) { return ErrorCode::CONNECTION_FAILED; } @@ -695,11 +338,16 @@ void PosixE3Connector::accept_pending() { if (fd < 0) { break; // Nothing more to accept now } - if (!admit(&address)) { + size_t connected; + { + std::lock_guard lock(peers_mutex_); + connected = peers_.size(); + } + if (!admit(spec(), address, connected)) { close(fd); continue; } - tune_socket(fd, SocketKind::ACCEPTED); + tune_socket(spec(), fd, SocketKind::ACCEPTED); auto peer = std::make_shared(fd, next_token_++); size_t count; { @@ -980,7 +628,7 @@ std::vector PosixE3Connector::take_closed_peers() { // =========================================================================== ErrorCode PosixE3Connector::setup_initial_connection_client() { - const Target where = target(); + const SocketSpec where = spec(); int fd = -1; for (int attempt = 0; attempt < CONNECT_ATTEMPTS && fd < 0; ++attempt) { fd = connect_to(where); @@ -992,7 +640,7 @@ ErrorCode PosixE3Connector::setup_initial_connection_client() { } } if (fd < 0) { - E3_LOG_ERROR(LOG_TAG) << "Failed to connect to " << setup_endpoint_ << " port " << where.port + E3_LOG_ERROR(LOG_TAG) << "Failed to connect to " << setup_endpoint_ << " port " << where.target.port << ": " << strerror(errno); return ErrorCode::CONNECTION_FAILED; } @@ -1002,7 +650,7 @@ ErrorCode PosixE3Connector::setup_initial_connection_client() { // thread must not take it for an indication. awaiting_setup_response_.store(true); connected_ = true; - E3_LOG_INFO(LOG_TAG) << "Connected to " << setup_endpoint_ << " port " << where.port; + E3_LOG_INFO(LOG_TAG) << "Connected to " << setup_endpoint_ << " port " << where.target.port; return ErrorCode::SUCCESS; } diff --git a/src/connector/posix_connector.hpp b/src/connector/posix_connector.hpp index 021feb5c..4a46cba5 100644 --- a/src/connector/posix_connector.hpp +++ b/src/connector/posix_connector.hpp @@ -10,6 +10,7 @@ #define LIBE3_POSIX_CONNECTOR_HPP #include "libe3/e3_connector.hpp" +#include "socket_link.hpp" #include #include #include @@ -100,25 +101,7 @@ class PosixE3Connector : public E3Connector { struct Peer; struct Queued; - /// Which end of a connection a socket is, as the options it needs differ. - enum class SocketKind { - LISTENER, ///< A RAN's listening socket - CLIENT, ///< A dApp's socket, before it connects - ACCEPTED ///< A RAN's socket for one accepted connection - }; - - /// Where the endpoint string says to listen or connect. - struct Target { - std::string host; ///< Empty: the default (any address to listen, loopback to connect) - std::string port; ///< Decimal port - }; - - Target target() const; - int make_socket(int family) const; - void tune_socket(int fd, SocketKind kind) const; - int listen_on(const Target& where); - int connect_to(const Target& where); - bool admit(const void* address) const; + socket_link::SocketSpec spec() const; void accept_pending(); std::shared_ptr find_peer(uint64_t token); bool pump(std::vector& out, uint64_t& token, int timeout_ms); diff --git a/src/connector/socket_link.cpp b/src/connector/socket_link.cpp new file mode 100644 index 00000000..591d47a0 --- /dev/null +++ b/src/connector/socket_link.cpp @@ -0,0 +1,446 @@ +/** + * @file socket_link.cpp + * @brief Endpoint, sockets and framing shared by the POSIX and the io_uring link + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "socket_link.hpp" +#include "libe3/logger.hpp" + +#include +#include +#include +#include +#include +#include +#ifdef LIBE3_ENABLE_SCTP +#include +#endif +#include +#include +#include +#include +#include +#include + +namespace libe3 { +namespace socket_link { + +namespace { + +/** + * @brief Create a socket of the SCTP protocol. + * + * SCTP is optional at build time (LIBE3_ENABLE_SCTP, off by default). The enumerator + * stays in the ABI either way, so asking for SCTP against a build without it has to + * fail loudly here rather than fall through to TCP, which would look as if it worked + * while putting traffic on the wrong protocol. Returns -1 with errno set, like socket(). + */ +int sctp_socket(int family, const char* log_tag) { +#ifdef LIBE3_ENABLE_SCTP + (void)log_tag; + return socket(family, SOCK_STREAM, IPPROTO_SCTP); +#else + (void)family; + E3_LOG_ERROR(log_tag) << "SCTP transport requested but libe3 was built without it. " + "Rebuild with -DLIBE3_ENABLE_SCTP=ON (needs libsctp-dev), " + "or use the TCP or IPC transport."; + errno = EPROTONOSUPPORT; + return -1; +#endif +} + +void set_timeout(int fd, int option, int timeout_ms, const char* name, const char* log_tag) { + struct timeval tv; + tv.tv_sec = timeout_ms / 1000; + tv.tv_usec = (timeout_ms % 1000) * 1000; + if (setsockopt(fd, SOL_SOCKET, option, &tv, sizeof(tv)) != 0) { + E3_LOG_WARN(log_tag) << "Failed to set " << name << ": " << strerror(errno); + } +} + +void set_nonblocking(int fd, const char* log_tag) { + int flags = fcntl(fd, F_GETFL, 0); + if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { + E3_LOG_WARN(log_tag) << "Failed to set O_NONBLOCK: " << strerror(errno); + } +} + +} // namespace + +std::string strip_scheme(const std::string& endpoint) { + auto pos = endpoint.find("://"); + return pos == std::string::npos ? endpoint : endpoint.substr(pos + 3); +} + +std::string link_endpoint(E3TransportLayer transport, const std::string& endpoint) { + if (transport == E3TransportLayer::IPC) { + return endpoint == SHARED_DEFAULT_ENDPOINT ? DEFAULT_IPC_PATH : strip_scheme(endpoint); + } + return endpoint; +} + +Target parse_target(const std::string& endpoint_uri, uint16_t port) { + Target where{"", std::to_string(port)}; + std::string rest; + for (const char* scheme : {"tcp://", "sctp://"}) { + if (endpoint_uri.rfind(scheme, 0) == 0) { + rest = endpoint_uri.substr(std::strlen(scheme)); + break; + } + } + if (rest.empty()) { + return where; + } + if (rest[0] == '[') { + const auto close = rest.find(']'); + if (close == std::string::npos) { + return where; + } + where.host = rest.substr(1, close - 1); + if (close + 1 < rest.size() && rest[close + 1] == ':') { + where.port = rest.substr(close + 2); + } + } else { + const auto colon = rest.rfind(':'); + if (colon != std::string::npos && rest.find(':') == colon) { + where.host = rest.substr(0, colon); + where.port = rest.substr(colon + 1); + } else { + where.host = rest; + } + } + if (where.port.empty()) { + where.port = std::to_string(port); + } + return where; +} + +int make_socket(const SocketSpec& spec, int family) { + switch (spec.transport) { + case E3TransportLayer::IPC: + return socket(AF_UNIX, SOCK_SEQPACKET, 0); + case E3TransportLayer::SCTP: + return sctp_socket(family, spec.log_tag); + default: + return socket(family, SOCK_STREAM, 0); + } +} + +void tune_socket(const SocketSpec& spec, int fd, SocketKind kind) { + const char* tag = spec.log_tag; + if (spec.transport == E3TransportLayer::IPC) { + if (kind == SocketKind::ACCEPTED) { + set_timeout(fd, SO_SNDTIMEO, SEND_TIMEOUT_MS, "SO_SNDTIMEO", tag); + } + return; + } + const int one = 1; + if (kind == SocketKind::LISTENER) { + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + } + if (kind != SocketKind::LISTENER) { + set_timeout(fd, SO_SNDTIMEO, SEND_TIMEOUT_MS, "SO_SNDTIMEO", tag); + } + if (spec.transport == E3TransportLayer::TCP) { + if (kind != SocketKind::LISTENER && setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)) != 0) { + E3_LOG_WARN(tag) << "Failed to set TCP_NODELAY: " << strerror(errno); + } + return; + } +#ifdef LIBE3_ENABLE_SCTP + if (kind != SocketKind::ACCEPTED) { + // Before the association exists, on both peers: two streams, and buffers and + // delayed SACK that do not stall a message larger than the Linux defaults allow. + struct sctp_initmsg init{}; + init.sinit_num_ostreams = 2; + init.sinit_max_instreams = 2; + if (setsockopt(fd, IPPROTO_SCTP, SCTP_INITMSG, &init, sizeof(init)) != 0) { + E3_LOG_WARN(tag) << "Failed to request two SCTP streams: " << strerror(errno); + } + if (spec.options.socket_buffer_bytes > 0) { + const int size = static_cast(spec.options.socket_buffer_bytes); + setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)); + setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)); + } + if (!spec.options.sctp_delayed_sack) { + struct sctp_sack_info sack{}; + sack.sack_assoc_id = 0; + sack.sack_delay = 0; + sack.sack_freq = 1; + if (setsockopt(fd, IPPROTO_SCTP, SCTP_DELAYED_SACK, &sack, sizeof(sack)) != 0) { + E3_LOG_WARN(tag) << "Failed to turn delayed SACK off: " << strerror(errno); + } + } + } + if (kind != SocketKind::LISTENER) { + if (setsockopt(fd, IPPROTO_SCTP, SCTP_NODELAY, &one, sizeof(one)) != 0) { + E3_LOG_WARN(tag) << "Failed to set SCTP_NODELAY: " << strerror(errno); + } + if (setsockopt(fd, IPPROTO_SCTP, SCTP_RECVRCVINFO, &one, sizeof(one)) != 0) { + E3_LOG_WARN(tag) << "Failed to ask for the SCTP receive info: " << strerror(errno); + } + } +#endif +} + +int listen_on(const SocketSpec& spec, bool nonblocking) { + const char* tag = spec.log_tag; + const Target& where = spec.target; + int fd = -1; + int result = -1; + if (spec.transport == E3TransportLayer::IPC) { + struct stat st{}; + if (stat(IPC_BASE_DIR, &st) == -1 && mkdir(IPC_BASE_DIR, 0777) == -1 && errno != EEXIST) { + E3_LOG_ERROR(tag) << "Failed to create IPC directory: " << strerror(errno); + return -1; + } + chmod(IPC_BASE_DIR, 0777); + if (spec.path.size() >= sizeof(sockaddr_un::sun_path)) { + E3_LOG_ERROR(tag) << "IPC path is too long: " << spec.path; + return -1; + } + fd = make_socket(spec, AF_UNIX); + if (fd < 0) { + return -1; + } + struct sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::memcpy(addr.sun_path, spec.path.c_str(), spec.path.size()); + unlink(spec.path.c_str()); + result = bind(fd, reinterpret_cast(&addr), sizeof(addr)); + } else if (where.host.empty()) { + // Any address, both families: a dual-stack socket, or IPv4 alone where there is no IPv6. + fd = make_socket(spec, AF_INET6); + if (fd >= 0) { + const int off = 0; + setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &off, sizeof(off)); + tune_socket(spec, fd, SocketKind::LISTENER); + struct sockaddr_in6 addr{}; + addr.sin6_family = AF_INET6; + addr.sin6_addr = in6addr_any; + addr.sin6_port = htons(static_cast(std::stoi(where.port))); + result = bind(fd, reinterpret_cast(&addr), sizeof(addr)); + if (result < 0) { + close(fd); + fd = -1; + } + } + if (fd < 0) { + fd = make_socket(spec, AF_INET); + if (fd < 0) { + return -1; + } + tune_socket(spec, fd, SocketKind::LISTENER); + struct sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + addr.sin_port = htons(static_cast(std::stoi(where.port))); + result = bind(fd, reinterpret_cast(&addr), sizeof(addr)); + } + } else { + struct addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + struct addrinfo* found = nullptr; + if (getaddrinfo(where.host.c_str(), where.port.c_str(), &hints, &found) != 0 || found == nullptr) { + E3_LOG_ERROR(tag) << "Cannot resolve " << where.host; + return -1; + } + fd = make_socket(spec, found->ai_family); + if (fd >= 0) { + tune_socket(spec, fd, SocketKind::LISTENER); + result = bind(fd, found->ai_addr, found->ai_addrlen); + } + freeaddrinfo(found); + if (fd < 0) { + return -1; + } + } + if (result < 0 || listen(fd, LISTEN_BACKLOG) < 0) { + E3_LOG_ERROR(tag) << "Failed to listen on " + << (spec.transport == E3TransportLayer::IPC ? spec.path : where.host + ":" + where.port) + << ": " << strerror(errno); + close(fd); + return -1; + } + if (nonblocking) { + set_nonblocking(fd, tag); + } + return fd; +} + +int connect_to(const SocketSpec& spec) { + if (spec.transport == E3TransportLayer::IPC) { + if (spec.path.size() >= sizeof(sockaddr_un::sun_path)) { + errno = ENAMETOOLONG; + return -1; + } + int fd = make_socket(spec, AF_UNIX); + if (fd < 0) { + return -1; + } + struct sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::memcpy(addr.sun_path, spec.path.c_str(), spec.path.size()); + if (connect(fd, reinterpret_cast(&addr), sizeof(addr)) < 0) { + const int saved = errno; + close(fd); + errno = saved; + return -1; + } + tune_socket(spec, fd, SocketKind::CLIENT); + return fd; + } + struct addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo* found = nullptr; + const std::string host = spec.target.host.empty() ? "127.0.0.1" : spec.target.host; + if (getaddrinfo(host.c_str(), spec.target.port.c_str(), &hints, &found) != 0 || found == nullptr) { + errno = EHOSTUNREACH; + return -1; + } + int fd = -1; + int saved = ECONNREFUSED; + for (struct addrinfo* a = found; a != nullptr && fd < 0; a = a->ai_next) { + int candidate = make_socket(spec, a->ai_family); + if (candidate < 0) { + saved = errno; + continue; + } + tune_socket(spec, candidate, SocketKind::CLIENT); + if (connect(candidate, a->ai_addr, a->ai_addrlen) == 0) { + fd = candidate; + } else { + saved = errno; + close(candidate); + } + } + freeaddrinfo(found); + if (fd < 0) { + errno = saved; + } + return fd; +} + +bool admit(const SocketSpec& spec, const sockaddr_storage& address, size_t connected) { + if (spec.transport == E3TransportLayer::IPC) { + return true; + } + if (!spec.options.allowed_peers.empty()) { + const std::string text = address_text(address); + if (std::find(spec.options.allowed_peers.begin(), spec.options.allowed_peers.end(), text) + == spec.options.allowed_peers.end()) { + E3_LOG_WARN(spec.log_tag) << "Refused a connection from " << text << ": not in the allowed peers"; + return false; + } + } + if (spec.options.maximum_associations > 0 && connected >= spec.options.maximum_associations) { + E3_LOG_WARN(spec.log_tag) << "Refused a connection from " << address_text(address) + << ": " << spec.options.maximum_associations << " peers are connected"; + return false; + } + return true; +} + +std::string address_text(const sockaddr_storage& ss) { + char text[INET6_ADDRSTRLEN] = {}; + if (ss.ss_family == AF_INET) { + const auto* in = reinterpret_cast(&ss); + inet_ntop(AF_INET, &in->sin_addr, text, sizeof(text)); + } else if (ss.ss_family == AF_INET6) { + const auto* in6 = reinterpret_cast(&ss); + if (IN6_IS_ADDR_V4MAPPED(&in6->sin6_addr)) { + inet_ntop(AF_INET, &in6->sin6_addr.s6_addr[12], text, sizeof(text)); + } else { + inet_ntop(AF_INET6, &in6->sin6_addr, text, sizeof(text)); + } + } + return text; +} + +StreamFrame parse_stream_frame(const uint8_t* data, size_t size, size_t& pdu_size) { + if (size < TCP_PREFIX_BYTES) { + return StreamFrame::NEED_MORE; + } + uint32_t length = 0; + std::memcpy(&length, data, sizeof(length)); + length = ntohl(length); + if (length == 0 || length > MAX_PDU) { + pdu_size = length; + return StreamFrame::INVALID; + } + if (size < TCP_PREFIX_BYTES + length) { + return StreamFrame::NEED_MORE; + } + pdu_size = length; + return StreamFrame::PDU; +} + +void write_stream_prefix(uint8_t (&prefix)[TCP_PREFIX_BYTES], size_t size) { + const uint32_t length = htonl(static_cast(size)); + std::memcpy(prefix, &length, sizeof(length)); +} + +void set_sctp_send_info(struct msghdr& msg, void* control, PduClass kind) { +#ifdef LIBE3_ENABLE_SCTP + static_assert(CMSG_SPACE(sizeof(struct sctp_sndinfo)) <= SCTP_SEND_CONTROL_BYTES, "control room"); + std::memset(control, 0, CMSG_SPACE(sizeof(struct sctp_sndinfo))); + msg.msg_control = control; + msg.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndinfo)); + struct cmsghdr* c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = IPPROTO_SCTP; + c->cmsg_type = SCTP_SNDINFO; + c->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndinfo)); + struct sctp_sndinfo info{}; + info.snd_sid = kind == PduClass::INDICATION ? SCTP_INDICATION_STREAM : SCTP_PROCEDURE_STREAM; + info.snd_ppid = htonl(SCTP_PPID); + std::memcpy(CMSG_DATA(c), &info, sizeof(info)); +#else + (void)msg; + (void)control; + (void)kind; +#endif +} + +bool sctp_is_notification(const struct msghdr& msg) { +#ifdef LIBE3_ENABLE_SCTP + return (msg.msg_flags & MSG_NOTIFICATION) != 0; +#else + (void)msg; + return false; +#endif +} + +bool sctp_ppid_acceptable(const struct msghdr& msg, const char* log_tag) { +#ifdef LIBE3_ENABLE_SCTP + static_assert(CMSG_SPACE(sizeof(struct sctp_rcvinfo)) <= SCTP_RECEIVE_CONTROL_BYTES, "control room"); + if (msg.msg_controllen == 0) { + return true; + } + auto& m = const_cast(msg); + for (struct cmsghdr* c = CMSG_FIRSTHDR(&m); c != nullptr; c = CMSG_NXTHDR(&m, c)) { + if (c->cmsg_level == IPPROTO_SCTP && c->cmsg_type == SCTP_RCVINFO) { + struct sctp_rcvinfo info; + std::memcpy(&info, CMSG_DATA(c), sizeof(info)); + const uint32_t ppid = ntohl(info.rcv_ppid); + if (ppid != 0 && ppid != SCTP_PPID) { + E3_LOG_ERROR(log_tag) << "Message with payload protocol id 0x" << std::hex << ppid; + return false; + } + } + } + return true; +#else + (void)msg; + (void)log_tag; + return true; +#endif +} + +} // namespace socket_link +} // namespace libe3 diff --git a/src/connector/socket_link.hpp b/src/connector/socket_link.hpp new file mode 100644 index 00000000..d3522cc1 --- /dev/null +++ b/src/connector/socket_link.hpp @@ -0,0 +1,198 @@ +/** + * @file socket_link.hpp + * @brief What the POSIX and the io_uring link share: the endpoint, the sockets and the framing + * + * Both links put the same bytes on the same sockets; they differ only in how they wait for + * and move those bytes. Keeping the endpoint parsing, the socket options, the admission rules + * and the framing here, once, is what makes a dApp of one link work with a RAN of the other. + * Internal: not installed. + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef LIBE3_SOCKET_LINK_HPP +#define LIBE3_SOCKET_LINK_HPP + +#include "libe3/e3_connector.hpp" +#include "libe3/types.hpp" + +#include +#include +#include +#include + +namespace libe3 { +namespace socket_link { + +constexpr const char* IPC_BASE_DIR = "/tmp/dapps"; ///< Made by a RAN that listens on IPC, open to all +/// What E3Config holds for setup_endpoint unless the program sets it. The ZeroMQ link shares +/// the string, so the socket links map it to their own default instead of changing it. +constexpr const char* SHARED_DEFAULT_ENDPOINT = "ipc:///tmp/dapps/setup"; +constexpr const char* DEFAULT_IPC_PATH = "/tmp/dapps/e3"; ///< The IPC endpoint of the socket links by default +constexpr int SEND_TIMEOUT_MS = 500; ///< A send that waits this long drops its peer +constexpr int CONNECT_ATTEMPTS = 10; ///< The dApp waits for a RAN that is not up yet +constexpr int CONNECT_PAUSE_MS = 200; ///< Between two of those attempts +constexpr int LISTEN_BACKLOG = 64; ///< Connections the kernel holds before they are accepted +constexpr size_t MAX_PDU = MAX_PROTOCOL_DATA_SIZE * 2; ///< One limit for every transport and direction +constexpr uint32_t SCTP_PPID = 0x00E3A001; ///< Project-private; no payload protocol id is registered for E3AP +constexpr int SCTP_PROCEDURE_STREAM = 0; ///< SCTP stream of everything but an indication +constexpr int SCTP_INDICATION_STREAM = 1; ///< SCTP stream of the indications a RAN sends +constexpr size_t TCP_PREFIX_BYTES = 4; ///< TCP: each PDU follows its length, big-endian + +/// Which end of a connection a socket is, as the options it needs differ. +enum class SocketKind { + LISTENER, ///< A RAN's listening socket + CLIENT, ///< A dApp's socket, before it connects + ACCEPTED ///< A RAN's socket for one accepted connection +}; + +/// Where the endpoint string says to listen or connect. +struct Target { + std::string host; ///< Empty: the default (any address to listen, loopback to connect) + std::string port; ///< Decimal port +}; + +/// How a socket is opened and tuned, the same for both links. +struct SocketSpec { + E3TransportLayer transport; ///< SCTP, TCP or IPC + std::string path; ///< IPC: the socket file + Target target; ///< TCP and SCTP: host and port + PosixOptions options; ///< Buffers, delayed SACK, admission + const char* log_tag; ///< The connector's log tag +}; + +/** + * @brief The endpoint without a URI scheme ("ipc://", "tcp://"). + * @param endpoint An endpoint, with or without a scheme + * @return What follows "://", or the endpoint as it is + */ +std::string strip_scheme(const std::string& endpoint); + +/** + * @brief The endpoint a socket link reports and uses: an IPC path, or the URI as given. + * + * IPC: the "ipc://" prefix is optional, and the library's shared default is replaced by + * /tmp/dapps/e3. + * @param transport The transport of the link + * @param endpoint The endpoint the program configured + * @return The IPC path, or for TCP and SCTP the endpoint unchanged + */ +std::string link_endpoint(E3TransportLayer transport, const std::string& endpoint); + +/** + * @brief Host and port of "tcp://host:port" or "sctp://host:port" (a host in brackets for IPv6). + * + * Anything else means the default host and @p port. + * @param endpoint_uri The endpoint the program configured + * @param port The port to use when the endpoint names none + * @return The host (empty for the default) and the port + */ +Target parse_target(const std::string& endpoint_uri, uint16_t port); + +/** + * @brief A socket of the link's kind; SCTP without the build support fails loudly. + * @param spec The link + * @param family AF_INET or AF_INET6 (IPC ignores it) + * @return The socket, or -1 with errno set + */ +int make_socket(const SocketSpec& spec, int family); + +/** + * @brief Set what a socket of @p kind needs: timeouts, no-delay, SCTP streams, buffers, SACK and timers. + * @param spec The link + * @param fd The socket + * @param kind Which end of a connection it is + */ +void tune_socket(const SocketSpec& spec, int fd, SocketKind kind); + +/** + * @brief Bind and listen on the endpoint, IPv4 and IPv6 alike when no host is named. + * @param spec The link + * @param nonblocking Set O_NONBLOCK on the listener (the POSIX link polls; io_uring must not have it) + * @return The listening socket, or -1 + */ +int listen_on(const SocketSpec& spec, bool nonblocking); + +/** + * @brief Connect to the endpoint once. + * @param spec The link + * @return The connected socket, or -1 with errno set + */ +int connect_to(const SocketSpec& spec); + +/** + * @brief Whether a peer that just connected from @p address may stay (RAN role). + * @param spec The link, with the peer list and the cap + * @param address Where the peer connected from + * @param connected How many peers are connected already, for the association cap + * @return true to keep the connection, false to close it + */ +bool admit(const SocketSpec& spec, const sockaddr_storage& address, size_t connected); + +/** + * @brief The text of a peer's address, with an IPv4-mapped IPv6 address shown as IPv4. + * @param address The peer's address + * @return Its literal text, or empty for a family other than IPv4 and IPv6 + */ +std::string address_text(const sockaddr_storage& address); + +/** @brief What the bytes at the front of a TCP stream hold. */ +enum class StreamFrame { + PDU, ///< A whole PDU: its length is in @p pdu_size, its bytes follow the prefix + NEED_MORE, ///< Not yet a whole PDU + INVALID ///< A length of zero or above MAX_PDU: the stream cannot be trusted any more +}; + +/** + * @brief Read the PDU at the front of buffered TCP bytes. + * @param data The bytes, prefix first + * @param size How many there are + * @param pdu_size Set to the PDU's length when the result is PDU (the frame is TCP_PREFIX_BYTES + it), + * and to the length the prefix gave when it is INVALID + * @return What the bytes hold + */ +StreamFrame parse_stream_frame(const uint8_t* data, size_t size, size_t& pdu_size); + +/** + * @brief The TCP prefix of a PDU of @p size bytes. + * @param prefix Where the four octets go + * @param size The length of the PDU + */ +void write_stream_prefix(uint8_t (&prefix)[TCP_PREFIX_BYTES], size_t size); + +/** @brief Room for the control message an SCTP send carries. */ +constexpr size_t SCTP_SEND_CONTROL_BYTES = 64; + +/** @brief Room for the control message an SCTP receive asks for. */ +constexpr size_t SCTP_RECEIVE_CONTROL_BYTES = 64; + +/** + * @brief Put the SCTP send info (payload protocol id, stream for @p kind) into @p msg. + * @param msg The message to send; its control fields are set + * @param control Where the control message is built; at least SCTP_SEND_CONTROL_BYTES, aligned for cmsghdr + * @param kind What the PDU is for, which picks the stream + */ +void set_sctp_send_info(struct msghdr& msg, void* control, PduClass kind); + +/** + * @brief Whether what an SCTP recvmsg returned is a notification of the stack, not data. + * @param msg The message as recvmsg filled it + * @return true for a notification + */ +bool sctp_is_notification(const struct msghdr& msg); + +/** + * @brief Whether a received SCTP message may be taken, by its payload protocol id. + * + * 0 and the id libe3 stamps are taken. A message without the receive info is taken too. + * @param msg The message as recvmsg filled it, with its control data + * @param log_tag Where to log a refusal + * @return false for another payload protocol id + */ +bool sctp_ppid_acceptable(const struct msghdr& msg, const char* log_tag); + +} // namespace socket_link +} // namespace libe3 + +#endif // LIBE3_SOCKET_LINK_HPP diff --git a/tests/test_socket_link.cpp b/tests/test_socket_link.cpp new file mode 100644 index 00000000..d636e882 --- /dev/null +++ b/tests/test_socket_link.cpp @@ -0,0 +1,120 @@ +/** + * @file test_socket_link.cpp + * @brief The framing and endpoint rules that the POSIX and the io_uring link share. + * + * Pure functions, no sockets, so it runs in every build. Properties verified: + * - a TCP frame is its length in four octets, network order, then the PDU; + * - a frame split anywhere needs more bytes, two frames back to back read as two; + * - a length of zero or above the limit is refused, the limit itself is taken; + * - the endpoint string gives host and port the same way for both links, IPv6 in brackets; + * - the shared default IPC endpoint of E3Config maps to /tmp/dapps/e3; + * - an SCTP socket of either link gets the bounded retransmission timeout, read back from the + * kernel (where the host has the sctp module): with Linux's own 1 s minimum, a burst of large + * PDUs dropped the dApp in 7 to 8 of 100 runs of 2000 PDUs on both links, and in none with it. + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "test_framework.hpp" +#include "connector/socket_link.hpp" + +#include +#include +#include +#include +#include +#include +#ifdef LIBE3_ENABLE_SCTP +#include +#include +#endif + +using namespace libe3; +using namespace libe3::socket_link; + +namespace { + +std::vector frame(size_t size, uint8_t fill) { + std::vector out(TCP_PREFIX_BYTES + size, fill); + uint8_t prefix[TCP_PREFIX_BYTES]; + write_stream_prefix(prefix, size); + std::memcpy(out.data(), prefix, TCP_PREFIX_BYTES); + return out; +} + +} // namespace + +TEST(the_prefix_is_four_octets_in_network_order) { + uint8_t prefix[TCP_PREFIX_BYTES]; + write_stream_prefix(prefix, 0x01020304); + ASSERT_EQ(prefix[0], 1); + ASSERT_EQ(prefix[1], 2); + ASSERT_EQ(prefix[2], 3); + ASSERT_EQ(prefix[3], 4); +} + +TEST(a_frame_split_anywhere_needs_more_and_whole_reads_as_its_pdu) { + const auto whole = frame(10, 0xAB); + size_t size = 0; + for (size_t cut = 0; cut < whole.size(); ++cut) { + ASSERT_TRUE(parse_stream_frame(whole.data(), cut, size) == StreamFrame::NEED_MORE); + } + ASSERT_TRUE(parse_stream_frame(whole.data(), whole.size(), size) == StreamFrame::PDU); + ASSERT_EQ(size, size_t{10}); +} + +TEST(two_frames_back_to_back_read_as_two) { + auto both = frame(3, 1); + const auto second = frame(5, 2); + both.insert(both.end(), second.begin(), second.end()); + size_t size = 0; + ASSERT_TRUE(parse_stream_frame(both.data(), both.size(), size) == StreamFrame::PDU); + ASSERT_EQ(size, size_t{3}); + const size_t next = TCP_PREFIX_BYTES + size; + ASSERT_TRUE(parse_stream_frame(both.data() + next, both.size() - next, size) == StreamFrame::PDU); + ASSERT_EQ(size, size_t{5}); + ASSERT_EQ(both[next + TCP_PREFIX_BYTES], 2); +} + +TEST(a_length_of_zero_or_above_the_limit_is_refused_and_the_limit_is_taken) { + size_t size = 0; + const uint8_t zero[TCP_PREFIX_BYTES] = {0, 0, 0, 0}; + ASSERT_TRUE(parse_stream_frame(zero, sizeof(zero), size) == StreamFrame::INVALID); + uint8_t over[TCP_PREFIX_BYTES]; + write_stream_prefix(over, MAX_PDU + 1); + ASSERT_TRUE(parse_stream_frame(over, sizeof(over), size) == StreamFrame::INVALID); + const auto at_limit = frame(MAX_PDU, 9); + ASSERT_TRUE(parse_stream_frame(at_limit.data(), at_limit.size(), size) == StreamFrame::PDU); + ASSERT_EQ(size, MAX_PDU); + ASSERT_EQ(MAX_PDU, size_t{65536}); +} + +TEST(the_endpoint_names_host_and_port_the_same_way_for_both_links) { + Target t = parse_target("tcp://10.0.0.1:1234", 9990); + ASSERT_EQ(t.host, std::string("10.0.0.1")); + ASSERT_EQ(t.port, std::string("1234")); + t = parse_target("sctp://[::1]:36423", 9990); + ASSERT_EQ(t.host, std::string("::1")); + ASSERT_EQ(t.port, std::string("36423")); + t = parse_target("sctp://[fe80::1]", 9990); + ASSERT_EQ(t.host, std::string("fe80::1")); + ASSERT_EQ(t.port, std::string("9990")); + t = parse_target("tcp://ran.example", 9990); + ASSERT_EQ(t.host, std::string("ran.example")); + ASSERT_EQ(t.port, std::string("9990")); + t = parse_target("ipc:///tmp/dapps/setup", 7); + ASSERT_TRUE(t.host.empty()); + ASSERT_EQ(t.port, std::string("7")); +} + +TEST(the_shared_default_ipc_endpoint_maps_to_e3) { + ASSERT_EQ(link_endpoint(E3TransportLayer::IPC, SHARED_DEFAULT_ENDPOINT), std::string("/tmp/dapps/e3")); + ASSERT_EQ(link_endpoint(E3TransportLayer::IPC, "ipc:///run/e3.sock"), std::string("/run/e3.sock")); + ASSERT_EQ(link_endpoint(E3TransportLayer::IPC, "/run/e3.sock"), std::string("/run/e3.sock")); + ASSERT_EQ(link_endpoint(E3TransportLayer::TCP, "tcp://h:1"), std::string("tcp://h:1")); +} + +int main() { + return RUN_ALL_TESTS(); +} From 113015083da6f98552ee4a984316100db8d6f033 Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Wed, 11 Nov 2026 15:25:59 -0500 Subject: [PATCH 2/7] connector: bound the retransmission timeout of every SCTP socket Linux gives an SCTP association a minimum retransmission timeout of one second. On loopback a burst of large PDUs can lose one packet to a full receive window, the association then waits that second, and the send stalls past the 500 ms send timeout: the RAN drops a dApp that is alive. A burst of 2000 PDUs of up to 64 KiB from the RAN, repeated 100 times, dropped the dApp 8 times with the POSIX link and 7 times with the io_uring link of the next commits, and never over TCP; with SCTP_RTOINFO set on the socket (initial 120 ms, minimum 20 ms, maximum 250 ms, the maximum kept under the send timeout) it happened 0 times in 100 with each link. This changes the timers of #153's POSIX SCTP sockets too, for the better as far as the runs show, but it is a choice about a transport's behaviour, which is why it is a commit of its own: dropping it removes the bound and nothing else. test_socket_link reads the values back from the socket and fails without it. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- src/connector/socket_link.cpp | 7 +++++++ src/connector/socket_link.hpp | 6 ++++++ tests/test_socket_link.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/src/connector/socket_link.cpp b/src/connector/socket_link.cpp index 591d47a0..00a32453 100644 --- a/src/connector/socket_link.cpp +++ b/src/connector/socket_link.cpp @@ -175,6 +175,13 @@ void tune_socket(const SocketSpec& spec, int fd, SocketKind kind) { } } } + struct sctp_rtoinfo rto{}; + rto.srto_initial = SCTP_RTO_INITIAL_MS; + rto.srto_min = SCTP_RTO_MIN_MS; + rto.srto_max = SCTP_RTO_MAX_MS; + if (setsockopt(fd, IPPROTO_SCTP, SCTP_RTOINFO, &rto, sizeof(rto)) != 0) { + E3_LOG_WARN(tag) << "Failed to set the SCTP retransmission timeout: " << strerror(errno); + } if (kind != SocketKind::LISTENER) { if (setsockopt(fd, IPPROTO_SCTP, SCTP_NODELAY, &one, sizeof(one)) != 0) { E3_LOG_WARN(tag) << "Failed to set SCTP_NODELAY: " << strerror(errno); diff --git a/src/connector/socket_link.hpp b/src/connector/socket_link.hpp index d3522cc1..7cdbeb5a 100644 --- a/src/connector/socket_link.hpp +++ b/src/connector/socket_link.hpp @@ -39,6 +39,12 @@ constexpr uint32_t SCTP_PPID = 0x00E3A001; ///< Project-private; no payload p constexpr int SCTP_PROCEDURE_STREAM = 0; ///< SCTP stream of everything but an indication constexpr int SCTP_INDICATION_STREAM = 1; ///< SCTP stream of the indications a RAN sends constexpr size_t TCP_PREFIX_BYTES = 4; ///< TCP: each PDU follows its length, big-endian +/// SCTP retransmission timeout of each association, in ms: first, smallest and largest. Linux's own +/// smallest, 1 s, is longer than SEND_TIMEOUT_MS, so a burst of large PDUs that closes the peer's +/// window could stall a send until the RAN dropped the dApp; E3 runs between neighbours. +constexpr uint32_t SCTP_RTO_INITIAL_MS = 120; +constexpr uint32_t SCTP_RTO_MIN_MS = 20; ///< See SCTP_RTO_INITIAL_MS +constexpr uint32_t SCTP_RTO_MAX_MS = 250; ///< Half SEND_TIMEOUT_MS: one backed-off wait cannot reach it /// Which end of a connection a socket is, as the options it needs differ. enum class SocketKind { diff --git a/tests/test_socket_link.cpp b/tests/test_socket_link.cpp index d636e882..8a0bd8ee 100644 --- a/tests/test_socket_link.cpp +++ b/tests/test_socket_link.cpp @@ -115,6 +115,32 @@ TEST(the_shared_default_ipc_endpoint_maps_to_e3) { ASSERT_EQ(link_endpoint(E3TransportLayer::TCP, "tcp://h:1"), std::string("tcp://h:1")); } +TEST(an_sctp_socket_gets_the_bounded_retransmission_timeout) { +#ifdef LIBE3_ENABLE_SCTP + for (SocketKind kind : {SocketKind::LISTENER, SocketKind::CLIENT, SocketKind::ACCEPTED}) { + SocketSpec spec{E3TransportLayer::SCTP, "", Target{"", "1"}, PosixOptions{}, "test"}; + const int fd = make_socket(spec, AF_INET); + if (fd < 0) { + std::cout << " SCTP NOT RUN: this host cannot open an SCTP socket\n"; + const char* require = std::getenv("LIBE3_REQUIRE_SCTP"); + ASSERT_FALSE(require != nullptr && *require != '\0' && *require != '0'); + return; + } + tune_socket(spec, fd, kind); + struct sctp_rtoinfo rto{}; + socklen_t length = sizeof(rto); + ASSERT_EQ(getsockopt(fd, IPPROTO_SCTP, SCTP_RTOINFO, &rto, &length), 0); + ASSERT_EQ(rto.srto_initial, SCTP_RTO_INITIAL_MS); + ASSERT_EQ(rto.srto_min, SCTP_RTO_MIN_MS); + ASSERT_EQ(rto.srto_max, SCTP_RTO_MAX_MS); + ASSERT_TRUE(SCTP_RTO_MAX_MS < static_cast(SEND_TIMEOUT_MS)); + close(fd); + } +#else + std::cout << " SCTP NOT RUN: built without LIBE3_ENABLE_SCTP\n"; +#endif +} + int main() { return RUN_ALL_TESTS(); } From 2d21ee9122ecdf64e6894700cda099e7412ac717 Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Wed, 11 Nov 2026 15:25:59 -0500 Subject: [PATCH 3/7] connector: add the io_uring link, E3LinkLayer::URING A third link beside ZeroMQ and POSIX: the POSIX link's endpoint, sockets and bytes (IPC, TCP and SCTP, one endpoint per dApp, the framing of the one-endpoint wire), driven through Linux io_uring. A RAN on one of the two socket links serves a dApp on the other. For a user the facade gains one enumerator: URING = 2, the string "uring", C API range 0..2, E3LinkLayer_URING in Python. No E3Config field and no E3Connector method is added and PosixOptions apply as they are. The ring is tuned by LIBE3_URING_* environment variables read when the connector is created (depth, registered files, registered buffers, a kernel submission thread, busy polling of the completion queue): deployment tuning, not ABI. The calling threads drive two rings, one for receives and one for sends, with no I/O thread of the connector's own, so send() returns when the kernel has the bytes, as it does with POSIX, and the SEND_DONE stamp compares row by row. A broadcast to several dApps is one batch of sends, so a dApp that stopped reading no longer delays the others for the send timeout. uring_ring.{hpp,cpp} is the only file that includes . The link is built when it can be: LIBE3_ENABLE_URING (default ON) needs Linux and liburing, and LIBE3_HAS_URING is 0 or 1 for what links libe3. Without liburing, or where the kernel refuses io_uring (Docker's default seccomp profile, kernel.io_uring_disabled, a kernel before 5.11), create_connector returns null and says why, as it does for ZeroMQ without libzmq: there is no silent fallback. build_libe3 learns --enable-uring and --disable-uring and installs liburing-dev with its other dependencies. The decisions, what was left out and why (per-thread rings with SINGLE_ISSUER and DEFER_TASKRUN, a thread of its own for I/O, a direct-send bypass, multishot receive) are in docs/connectors/uring.md. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- build_libe3 | 20 + cmake/libe3Config.cmake.in | 8 + cmake/libe3Dependencies.cmake | 23 + cmake/libe3Options.cmake | 6 + cmake/libe3Sources.cmake | 8 + cmake/libe3Targets.cmake | 16 + include/libe3/c_api.h | 4 +- include/libe3/e3_connector.hpp | 17 +- include/libe3/types.hpp | 13 +- src/c_api.cpp | 2 +- src/connector/connector_factory.cpp | 19 + src/connector/uring_connector.cpp | 1290 +++++++++++++++++++++++++++ src/connector/uring_connector.hpp | 231 +++++ src/connector/uring_ring.cpp | 287 ++++++ src/connector/uring_ring.hpp | 224 +++++ tests/test_swig_smoke.py | 6 + tests/test_types.cpp | 2 + 17 files changed, 2162 insertions(+), 14 deletions(-) create mode 100644 src/connector/uring_connector.cpp create mode 100644 src/connector/uring_connector.hpp create mode 100644 src/connector/uring_ring.cpp create mode 100644 src/connector/uring_ring.hpp diff --git a/build_libe3 b/build_libe3 index 8bdde78c..7e57ec72 100755 --- a/build_libe3 +++ b/build_libe3 @@ -35,6 +35,7 @@ ENABLE_ASN1=1 ENABLE_JSON=1 ENABLE_PROTOBUF=0 ENABLE_SCTP=0 +ENABLE_URING=1 ENABLE_ASAN=0 ENABLE_TSAN=0 ENABLE_SWIG=0 @@ -121,6 +122,11 @@ Features: --install-deps does NOT install them; SCTP is optional and the deployed E3 links are IPC or TCP. --disable-sctp Disable the SCTP transport + --enable-uring Build the io_uring link (--link uring) when it can be built: on Linux + with liburing (default: ON). Without liburing the build goes on + without the link. --install-deps installs liburing-dev + (Debian/Ubuntu), liburing-devel (Fedora/RHEL) or liburing (Arch). + --disable-uring Never build the io_uring link --enable-swig Build the Python (SWIG) binding, libe3py, in build/swig to develop and test it (default: OFF). It is not installed with the library: install it with pip (pip install libe3py, or pip install . in this checkout) @@ -295,6 +301,7 @@ install_dependencies() { cmake \ pkg-config \ libzmq3-dev \ + liburing-dev \ ninja-build \ git \ autoconf \ @@ -318,6 +325,7 @@ install_dependencies() { cmake \ pkg-config \ zeromq-devel \ + liburing-devel \ ninja-build \ git \ asn1c \ @@ -333,6 +341,7 @@ install_dependencies() { cmake \ pkg-config \ zeromq \ + liburing \ ninja \ git \ asn1c \ @@ -346,6 +355,7 @@ install_dependencies() { echo " - C++17 compiler (gcc >= 7, clang >= 5)" echo " - pkg-config" echo " - libzmq (optional, for ZMQ transport)" + echo " - liburing >= 2.0 (optional, Linux, for the io_uring link)" echo " - ninja (optional, for faster builds)" exit 1 fi @@ -386,6 +396,7 @@ configure_cmake() { "-DLIBE3_ENABLE_JSON=$([ $ENABLE_JSON -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_PROTOBUF=$([ $ENABLE_PROTOBUF -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_SCTP=$([ $ENABLE_SCTP -eq 1 ] && echo ON || echo OFF)" + "-DLIBE3_ENABLE_URING=$([ $ENABLE_URING -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_ASAN=$([ $ENABLE_ASAN -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_TSAN=$([ $ENABLE_TSAN -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_SWIG=$([ $ENABLE_SWIG -eq 1 ] && echo ON || echo OFF)" @@ -612,6 +623,7 @@ print_summary() { echo " Tests: $([ $BUILD_TESTS -eq 1 ] && echo 'enabled' || echo 'disabled')" echo " Examples: $([ $BUILD_EXAMPLES -eq 1 ] && echo 'enabled' || echo 'disabled')" echo " ZMQ Support: $([ $ENABLE_ZMQ -eq 1 ] && echo 'enabled' || echo 'disabled')" + echo " io_uring Link: $([ $ENABLE_URING -eq 1 ] && echo 'when liburing is found' || echo 'disabled')" echo " ASN.1 Encoding: $([ $ENABLE_ASN1 -eq 1 ] && echo 'enabled' || echo 'disabled')" echo " AddressSan: $([ $ENABLE_ASAN -eq 1 ] && echo 'enabled' || echo 'disabled')" echo " ThreadSan: $([ $ENABLE_TSAN -eq 1 ] && echo 'enabled' || echo 'disabled')" @@ -715,6 +727,14 @@ main() { ENABLE_SCTP=0 shift ;; + --enable-uring) + ENABLE_URING=1 + shift + ;; + --disable-uring) + ENABLE_URING=0 + shift + ;; --disable-protobuf) ENABLE_PROTOBUF=0 shift diff --git a/cmake/libe3Config.cmake.in b/cmake/libe3Config.cmake.in index f797c37d..fe9ddaab 100644 --- a/cmake/libe3Config.cmake.in +++ b/cmake/libe3Config.cmake.in @@ -14,6 +14,14 @@ if(@LIBE3_ENABLE_ZMQ@) endif() endif() +# The static library links liburing privately, so a consumer of libe3::libe3 needs it to link. +if(@LIBE3_WITH_URING@) + find_dependency(PkgConfig) + if(PkgConfig_FOUND) + pkg_check_modules(URING IMPORTED_TARGET liburing) + endif() +endif() + # nlohmann/json is intentionally NOT a find_dependency: it is header-only and # used only inside libe3's implementation (compiled into the library, scoped to # the BUILD interface in libe3Targets.cmake), so consumers of the installed diff --git a/cmake/libe3Dependencies.cmake b/cmake/libe3Dependencies.cmake index b8aaafd3..186215f5 100644 --- a/cmake/libe3Dependencies.cmake +++ b/cmake/libe3Dependencies.cmake @@ -67,6 +67,29 @@ if(LIBE3_ENABLE_SCTP) endif() endif() +# Optional: liburing, for the io_uring link. Auto: a missing liburing, or a system other than +# Linux, leaves the link out with a status line instead of failing the configure. +set(LIBE3_WITH_URING OFF CACHE INTERNAL "The io_uring link is built") +if(LIBE3_ENABLE_URING) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(STATUS "io_uring link: not built (io_uring is Linux only)") + else() + find_package(PkgConfig QUIET) + if(PkgConfig_FOUND) + pkg_check_modules(URING QUIET IMPORTED_TARGET liburing>=2.0) + endif() + if(URING_FOUND) + set(LIBE3_WITH_URING ON CACHE INTERNAL "The io_uring link is built") + message(STATUS "io_uring link: built (liburing ${URING_VERSION})") + else() + message(STATUS "io_uring link: not built (liburing >= 2.0 not found; install liburing-dev " + "or liburing-devel, or configure with -DLIBE3_ENABLE_URING=OFF to silence this)") + endif() + endif() +else() + message(STATUS "io_uring link: not built (LIBE3_ENABLE_URING=OFF)") +endif() + # Optional: Google Benchmark for the integration micro-benchmarks # (dev-only; never linked into the shipped library) if(LIBE3_BUILD_INTEGRATION_TESTS) diff --git a/cmake/libe3Options.cmake b/cmake/libe3Options.cmake index f26a9ff7..1b49ef1d 100644 --- a/cmake/libe3Options.cmake +++ b/cmake/libe3Options.cmake @@ -19,6 +19,12 @@ option(LIBE3_ENABLE_ZMQ "Enable ZeroMQ transport" ON) # build with this OFF fails at socket creation with a clear message rather than # silently doing something else. option(LIBE3_ENABLE_SCTP "Enable the SCTP POSIX transport (needs libsctp-dev)" OFF) +# The io_uring link (E3LinkLayer::URING): the POSIX link's wire, driven through io_uring. +# ON means "when it can be built": on Linux with liburing found (liburing-dev, liburing-devel) +# it is built, anywhere else the configure says so and goes on without it, the way a build +# without libzmq goes on without the ZeroMQ link. OFF never builds it. The outcome is +# LIBE3_WITH_URING (see libe3Dependencies.cmake), and LIBE3_HAS_URING=0/1 for the code. +option(LIBE3_ENABLE_URING "Build the io_uring link when Linux and liburing allow it" ON) option(LIBE3_ENABLE_ASN1 "Enable ASN.1 encoding support" ON) # The E2SM-DAPP APER codec (libe3::e2sm_dapp): standalone libraries that carry # their own asn1c runtime and depend on nothing else in libe3. It is generated by diff --git a/cmake/libe3Sources.cmake b/cmake/libe3Sources.cmake index f35e4b9e..6432b33b 100644 --- a/cmake/libe3Sources.cmake +++ b/cmake/libe3Sources.cmake @@ -49,6 +49,14 @@ if(LIBE3_ENABLE_ZMQ) list(APPEND LIBE3_SOURCES src/connector/zmq_connector.cpp) endif() +# The io_uring connector; uring_ring.cpp is the only file that includes . +if(LIBE3_WITH_URING) + list(APPEND LIBE3_SOURCES + src/connector/uring_connector.cpp + src/connector/uring_ring.cpp + ) +endif() + # Backs latrec.h's TLS convenience API; excluded when the recorder is off, where # latrec.h falls back to inline no-op stubs. See docs/utilities/latrec.md. if(LIBE3_ENABLE_LATREC) diff --git a/cmake/libe3Targets.cmake b/cmake/libe3Targets.cmake index af84a102..1f73effa 100644 --- a/cmake/libe3Targets.cmake +++ b/cmake/libe3Targets.cmake @@ -82,6 +82,15 @@ if(LIBE3_ENABLE_SCTP) target_compile_definitions(libe3 PUBLIC LIBE3_ENABLE_SCTP) endif() +# As for ZeroMQ: always 0 or 1, PUBLIC so a consumer can tell whether E3LinkLayer::URING will +# work, and liburing PRIVATE (a static libe3 carries it as a link-only dependency). +if(LIBE3_WITH_URING) + target_compile_definitions(libe3 PUBLIC LIBE3_HAS_URING=1) + target_link_libraries(libe3 PRIVATE PkgConfig::URING) +else() + target_compile_definitions(libe3 PUBLIC LIBE3_HAS_URING=0) +endif() + if(LIBE3_ENABLE_LATREC) # PUBLIC: any consumer building against this target (OAI, flexric) must # see the same define, or its own latrec_* calls silently resolve to the @@ -169,6 +178,13 @@ if(LIBE3_ENABLE_SCTP) target_compile_definitions(libe3_shared PUBLIC LIBE3_ENABLE_SCTP) endif() +if(LIBE3_WITH_URING) + target_compile_definitions(libe3_shared PUBLIC LIBE3_HAS_URING=1) + target_link_libraries(libe3_shared PRIVATE PkgConfig::URING) +else() + target_compile_definitions(libe3_shared PUBLIC LIBE3_HAS_URING=0) +endif() + if(LIBE3_ENABLE_LATREC) target_compile_definitions(libe3_shared PUBLIC LIBE3_ENABLE_LATREC) if(LATREC_DEFAULT_DIR) diff --git a/include/libe3/c_api.h b/include/libe3/c_api.h index 42f526b4..aafd2b61 100644 --- a/include/libe3/c_api.h +++ b/include/libe3/c_api.h @@ -206,14 +206,14 @@ e3_error_t e3_service_model_emit_control_response( * All string pointers are copied by the library; the caller may free them * after the call. Use NULL for optional strings to keep library defaults. * - * Enum values: link_layer: 0=ZMQ, 1=POSIX. transport_layer: 0=SCTP, 1=TCP, 2=IPC. + * Enum values: link_layer: 0=ZMQ, 1=POSIX, 2=URING. transport_layer: 0=SCTP, 1=TCP, 2=IPC. * encoding: 0=ASN1, 1=JSON. Use -1 for link_layer, transport_layer, encoding, log_level * to keep defaults. Use 0 for setup_port, subscriber_port, publisher_port to keep defaults. * Set `set_dapp_idle_timeout` to apply `dapp_idle_timeout_s`; a zeroed struct keeps the default. */ typedef struct { const char* ran_identifier; ///< RAN identifier; NULL keeps the default - int link_layer; ///< 0=ZMQ, 1=POSIX, -1=default + int link_layer; ///< 0=ZMQ, 1=POSIX, 2=URING (io_uring), -1=default int transport_layer; ///< 0=SCTP, 1=TCP, 2=IPC, -1=default uint16_t setup_port; ///< Port of the setup channel (SCTP and TCP); 0 keeps the default uint16_t subscriber_port; ///< Port of the subscriber channel (SCTP and TCP); 0 keeps the default diff --git a/include/libe3/e3_connector.hpp b/include/libe3/e3_connector.hpp index 6f921665..ecac9f16 100644 --- a/include/libe3/e3_connector.hpp +++ b/include/libe3/e3_connector.hpp @@ -34,7 +34,7 @@ enum class PduClass : uint8_t { }; /** - * @brief Socket and admission options of the POSIX link + * @brief Socket and admission options of the POSIX and the io_uring link * * The ZeroMQ link ignores all of them. The defaults are those of E3Config. */ @@ -59,6 +59,8 @@ struct PosixOptions { * Derived classes: * - PosixE3Connector: POSIX socket-based transport (TCP, SCTP, IPC); one * endpoint per dApp carries the setup, inbound and outbound traffic + * - UringE3Connector: the same sockets and wire as PosixE3Connector, with + * the waiting and the sends done through io_uring (Linux, liburing) * - ZmqE3Connector: ZeroMQ-based transport */ class E3Connector { @@ -313,7 +315,7 @@ class E3Connector { * * Creates a connector instance based on the link and transport layers specified. * - * @param link_layer Link layer to use (ZMQ or POSIX) + * @param link_layer Link layer to use (ZMQ, POSIX or URING) * @param transport_layer Transport layer to use (SCTP, TCP, or IPC) * @param setup_endpoint Endpoint for setup channel * @param inbound_endpoint Endpoint for inbound channel @@ -323,11 +325,14 @@ class E3Connector { * @param outbound_port Port of the outbound channel (SCTP and TCP) * @param io_threads Number of I/O threads (for ZMQ) * @param role Role of the peer (RAN or dApp) - * @param posix Socket and admission options of the POSIX link (ignored by ZMQ) - * @return Unique pointer to created connector, nullptr on failure + * @param posix Socket and admission options of the POSIX and io_uring links (ignored by ZMQ) + * @return Unique pointer to created connector, nullptr on failure: the link was not built + * (ZMQ without libzmq, URING without liburing or off Linux), or the kernel refuses + * io_uring here (seccomp, kernel.io_uring_disabled, a kernel before 5.11); the + * reason is logged * - * The POSIX link serves everything on the setup endpoint and port, and ignores - * the inbound and outbound ones. + * The POSIX and io_uring links serve everything on the setup endpoint and port, and + * ignore the inbound and outbound ones. */ std::unique_ptr create_connector( E3LinkLayer link_layer, diff --git a/include/libe3/types.hpp b/include/libe3/types.hpp index b7ee8e4a..bbee8fca 100644 --- a/include/libe3/types.hpp +++ b/include/libe3/types.hpp @@ -61,7 +61,8 @@ enum class EncodingFormat : uint8_t { */ enum class E3LinkLayer : uint8_t { ZMQ = 0, ///< ZeroMQ-based transport - POSIX = 1 ///< POSIX socket-based transport + POSIX = 1, ///< POSIX socket-based transport + URING = 2 ///< The POSIX link's wire driven through io_uring (Linux, built with liburing) }; /** @@ -523,14 +524,15 @@ struct E3Config { // it would make every default-constructed config fail to initialise. The // POSIX connector accepts the same ZMQ-style endpoint URIs (it strips the // scheme), so the endpoint defaults below need no counterpart. - E3LinkLayer link_layer{E3LinkLayer::POSIX}; ///< Link-layer transport (ZMQ or POSIX) + E3LinkLayer link_layer{E3LinkLayer::POSIX}; ///< Link-layer transport (ZMQ, POSIX or URING) #else - E3LinkLayer link_layer{E3LinkLayer::ZMQ}; ///< Link-layer transport (ZMQ or POSIX) + E3LinkLayer link_layer{E3LinkLayer::ZMQ}; ///< Link-layer transport (ZMQ, POSIX or URING) #endif E3TransportLayer transport_layer{E3TransportLayer::IPC}; ///< Transport-layer protocol (SCTP, TCP, IPC) // Ports. The POSIX link serves everything on setup_port and ignores the other two; - // the ZeroMQ link uses all three. + // the ZeroMQ link uses all three. The io_uring link is the POSIX link's wire: whatever + // this structure says of the POSIX link holds for it too. uint16_t setup_port{9990}; ///< Port of the setup channel; the one port of the POSIX link uint16_t subscriber_port{9999}; ///< Port of the inbound control/subscription channel (ZeroMQ) uint16_t publisher_port{9991}; ///< Port of the outbound indication/publication channel (ZeroMQ) @@ -859,12 +861,13 @@ inline const char* role_to_string(E3Role role) noexcept { /** * @brief Convert E3LinkLayer to string representation * @param layer Link layer - * @return "zmq" or "posix", "unknown" for a value outside the enumeration + * @return "zmq", "posix" or "uring", "unknown" for a value outside the enumeration */ inline const char* link_layer_to_string(E3LinkLayer layer) noexcept { switch (layer) { case E3LinkLayer::ZMQ: return "zmq"; case E3LinkLayer::POSIX: return "posix"; + case E3LinkLayer::URING: return "uring"; default: return "unknown"; } } diff --git a/src/c_api.cpp b/src/c_api.cpp index 0db4cc63..10af84e5 100644 --- a/src/c_api.cpp +++ b/src/c_api.cpp @@ -236,7 +236,7 @@ e3_agent_handle_t* e3_agent_create_with_config(const e3_config_t* config) { E3Config cfg; if (config) { if (config->ran_identifier) cfg.ran_identifier = config->ran_identifier; - if (config->link_layer >= 0 && config->link_layer <= 1) + if (config->link_layer >= 0 && config->link_layer <= 2) cfg.link_layer = static_cast(config->link_layer); if (config->transport_layer >= 0 && config->transport_layer <= 2) cfg.transport_layer = static_cast(config->transport_layer); diff --git a/src/connector/connector_factory.cpp b/src/connector/connector_factory.cpp index 7767c188..a802dc57 100644 --- a/src/connector/connector_factory.cpp +++ b/src/connector/connector_factory.cpp @@ -11,6 +11,9 @@ #include "zmq_connector.hpp" #endif #include "posix_connector.hpp" +#if LIBE3_HAS_URING +#include "uring_connector.hpp" +#endif #include "libe3/logger.hpp" namespace libe3 { @@ -53,6 +56,22 @@ std::unique_ptr create_connector( conn = std::make_unique(transport_layer, setup_endpoint, setup_port, posix); break; + case E3LinkLayer::URING: +#if LIBE3_HAS_URING + // The POSIX link's endpoint and wire. Null when the kernel refuses io_uring; it logs why. + conn = UringE3Connector::create(transport_layer, setup_endpoint, setup_port, posix); + if (!conn) { + return nullptr; + } + break; +#else + // As for ZMQ above: the value can arrive from a configuration the build knows nothing about. + E3_LOG_ERROR("ConnFactory") + << "io_uring link layer requested but libe3 was built without it (LIBE3_ENABLE_URING=OFF, " + "liburing not found, or not Linux). Use the POSIX link, which speaks the same wire."; + return nullptr; +#endif + default: E3_LOG_ERROR("ConnFactory") << "Unsupported link layer: " << static_cast(link_layer); diff --git a/src/connector/uring_connector.cpp b/src/connector/uring_connector.cpp new file mode 100644 index 00000000..892a38b8 --- /dev/null +++ b/src/connector/uring_connector.cpp @@ -0,0 +1,1290 @@ +/** + * @file uring_connector.cpp + * @brief io_uring connector implementation + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "uring_connector.hpp" +#include "uring_ring.hpp" +#include "libe3/logger.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace libe3 { + +using namespace socket_link; + +namespace { + +constexpr const char* LOG_TAG = "UringConn"; +constexpr int RECEIVE_TIMEOUT_MS = 500; // As long as the POSIX link's receive waits +constexpr int SETUP_PUMP_MS = 100; +constexpr int64_t SETUP_PUMP_AFTER_NS = 1'000'000'000; // The setup thread reads if receive() has been idle this long +constexpr unsigned SLOT_COUNT = 256; // Connections beyond it use plain descriptors +constexpr size_t STREAM_CAPACITY = 2 * (MAX_PDU + TCP_PREFIX_BYTES); // TCP: room for two whole frames +constexpr size_t MESSAGE_CAPACITY = MAX_PDU + 1; // One byte more than a PDU may have + +// user_data: what the operation is in the top byte, the connection or send below it. +enum Op : uint64_t { OP_ACCEPT = 1, OP_RECV = 2, OP_WAKE = 3, OP_SEND = 4, OP_CANCEL = 5 }; +constexpr uint64_t ID_MASK = (uint64_t{1} << 56) - 1; + +inline uint64_t data_for(Op op, uint64_t id) { return (static_cast(op) << 56) | (id & ID_MASK); } +inline Op op_of(uint64_t data) { return static_cast(data >> 56); } +inline uint64_t id_of(uint64_t data) { return data & ID_MASK; } + +int64_t now_ns() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +bool read_number(const char* name, long low, long high, long& out) { + const char* text = std::getenv(name); + if (text == nullptr || *text == '\0') { + return false; + } + char* end = nullptr; + errno = 0; + const long value = std::strtol(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || value < low || value > high) { + E3_LOG_WARN(LOG_TAG) << "Ignoring " << name << "=" << text << ": expected a number from " + << low << " to " << high; + return false; + } + out = value; + return true; +} + +/// What a completed receive gave. +enum class Got { MORE, CLOSED, FAILED }; + +} // namespace + +// =========================================================================== +// Settings +// =========================================================================== + +UringE3Connector::Settings UringE3Connector::Settings::from_environment() { + Settings s; + long value = 0; + if (read_number("LIBE3_URING_DEPTH", 4, 4096, value)) s.depth = static_cast(value); + if (read_number("LIBE3_URING_FIXED_FILES", 0, 1, value)) s.fixed_files = value != 0; + if (read_number("LIBE3_URING_FIXED_BUFFERS", 0, 1, value)) s.fixed_buffers = value != 0; + if (read_number("LIBE3_URING_SQPOLL", 0, 1, value)) s.sqpoll = value != 0; + if (read_number("LIBE3_URING_SQPOLL_CPU", -1, 4095, value)) s.sqpoll_cpu = static_cast(value); + if (read_number("LIBE3_URING_BUSY_POLL_US", 0, 10'000'000, value)) s.busy_poll_us = static_cast(value); + return s; +} + +std::string UringE3Connector::Settings::describe() const { + std::ostringstream out; + out << "depth " << depth << ", registered files " << (fixed_files ? "on" : "off") + << ", registered buffers " << (fixed_buffers ? "on" : "off") + << ", kernel submission thread " << (sqpoll ? "on" : "off"); + if (sqpoll && sqpoll_cpu >= 0) out << " (CPU " << sqpoll_cpu << ")"; + out << ", busy poll " << busy_poll_us << " us"; + return out.str(); +} + +// =========================================================================== +// Connections, slots and queued PDUs +// =========================================================================== + +/** @brief Registered-file and buffer slots, shared with the connections that hold them. */ +struct UringE3Connector::Slots { + std::mutex mu; ///< Guards everything below + UringRing* rx{nullptr}; ///< The receive ring; null once the rings are gone + UringRing* tx{nullptr}; ///< The send ring; null once the rings are gone + bool files{false}; ///< Connections take a registered-file slot + bool buffers{false}; ///< Connections register their receive buffer + std::vector free; ///< Slots no connection holds + + /** + * @brief Take a free slot. + * @return The slot, or -1 when all are taken + */ + int take() { + std::lock_guard lock(mu); + if (free.empty()) return -1; + const int slot = free.back(); + free.pop_back(); + return slot; + } +}; + +/** @brief One connection. The reader alone touches the receive state. Closes its socket when the last reference goes. */ +struct UringE3Connector::Conn { + int fd{-1}; ///< The socket + uint64_t token{0}; ///< Never reused by this connector + E3TransportLayer transport{E3TransportLayer::TCP}; ///< The connector's transport + std::shared_ptr slots; ///< Where its slot goes back to + int slot{-1}; ///< Taken from slots, or -1 + int file_slot{-1}; ///< Registered in both rings, or -1 + int buffer_slot{-1}; ///< The receive buffer is registered there, or -1 + std::unique_ptr rx; ///< The receive buffer + size_t capacity{0}; ///< Its size + size_t held{0}; ///< TCP: stream bytes kept; SCTP: bytes of the message joined so far + bool recv_in_flight{false}; ///< A receive is armed on the ring + bool seen_first{false}; ///< The first PDU, the setup request, was read + bool closed{false}; ///< Dropped, or the peer closed + struct iovec iov{}; ///< SCTP: the receive's buffer + struct msghdr msg{}; ///< SCTP: the receive's header; the kernel writes flags and control back + alignas(struct cmsghdr) char control[SCTP_RECEIVE_CONTROL_BYTES]{}; ///< SCTP: the receive info + + /** + * @brief The socket as the rings name it. + * @return Its descriptor and registered-file slot + */ + RingFile file() const { return RingFile{fd, file_slot}; } + + /** @brief Empty its slots in the rings that are still there, and close the socket. */ + ~Conn() { + if (slot >= 0 && slots) { + std::lock_guard lock(slots->mu); + if (slots->rx != nullptr) { + if (file_slot >= 0) slots->rx->set_file(static_cast(file_slot), -1); + if (buffer_slot >= 0) slots->rx->set_buffer(static_cast(buffer_slot), nullptr, 0); + } + if (slots->tx != nullptr && file_slot >= 0) { + slots->tx->set_file(static_cast(file_slot), -1); + } + slots->free.push_back(slot); + } + if (fd >= 0) { + ::close(fd); + } + } +}; + +/** @brief A PDU read and not yet taken, or the close of a connection. */ +struct UringE3Connector::Ready { + std::shared_ptr conn; ///< Setup queue: who to answer + uint64_t token{0}; ///< Who sent it + std::vector data; ///< The PDU + bool closed{false}; ///< The connection closed after the PDUs before this one +}; + +/** @brief One write in flight, kept in place until the kernel completes it. */ +struct UringE3Connector::SendOp { + std::shared_ptr conn; ///< Where it goes + size_t done{0}; ///< Bytes the kernel took so far, prefix included + int state{0}; ///< 0 in flight, 1 sent, 2 failed + int error{0}; ///< Why it failed + struct iovec iov[2]{}; ///< What is left to write: the prefix, then the PDU + struct msghdr msg{}; ///< The sendmsg header + alignas(struct cmsghdr) char control[SCTP_SEND_CONTROL_BYTES]{}; ///< SCTP: the send info +}; + +// =========================================================================== +// Creation and teardown +// =========================================================================== + +UringE3Connector::UringE3Connector(E3TransportLayer transport_layer, const std::string& endpoint, uint16_t port, + const PosixOptions& options, const Settings& settings) + : transport_layer_(transport_layer) + , endpoint_uri_(endpoint) + , port_(port) + , options_(options) + , settings_(settings) +{ + setup_endpoint_ = link_endpoint(transport_layer, endpoint); +} + +std::unique_ptr UringE3Connector::create( + E3TransportLayer transport_layer, const std::string& endpoint, uint16_t port, const PosixOptions& options) { + std::unique_ptr conn( + new UringE3Connector(transport_layer, endpoint, port, options, Settings::from_environment())); + std::string error; + int code = 0; + if (!conn->open(&error, &code)) { + std::string hint; + if (code == EPERM || code == EACCES) { + hint = "io_uring is not permitted for this process: a seccomp profile refuses it (Docker's default " + "does: run with --security-opt seccomp=unconfined, or a profile that allows io_uring_setup, " + "io_uring_enter and io_uring_register), or the sysctl kernel.io_uring_disabled is 1 or 2"; + } else if (code == ENOSYS) { + hint = "this kernel has no io_uring, or one older than Linux 5.11"; + } else if (code == ENOMEM) { + hint = "the rings could not be locked in memory (RLIMIT_MEMLOCK)"; + } + E3_LOG_ERROR(LOG_TAG) << "Cannot use the io_uring link: " << error << (hint.empty() ? "" : ". ") << hint + << ". The POSIX link speaks the same wire and needs none of this."; + return nullptr; + } + E3_LOG_INFO(LOG_TAG) << "Creating io_uring connector, one endpoint: " << conn->setup_endpoint_ << " port " + << port << "; rings: " << conn->settings_.describe(); + return conn; +} + +bool UringE3Connector::open(std::string* error, int* error_code) { + RingOptions options; + options.depth = settings_.depth; + options.sqpoll = settings_.sqpoll; + options.sqpoll_cpu = settings_.sqpoll_cpu; + options.file_slots = settings_.fixed_files ? SLOT_COUNT : 0; + options.buffer_slots = settings_.fixed_buffers ? SLOT_COUNT : 0; + rx_ = UringRing::create(options, nullptr, error, error_code); + if (!rx_ && options.sqpoll) { + E3_LOG_WARN(LOG_TAG) << "The kernel refused a submission thread (" << *error << "); going on without one"; + options.sqpoll = false; + settings_.sqpoll = false; + rx_ = UringRing::create(options, nullptr, error, error_code); + } + if (!rx_) { + return false; + } + RingOptions send_options = options; + send_options.buffer_slots = 0; + tx_ = UringRing::create(send_options, rx_.get(), error, error_code); + if (!tx_ && send_options.sqpoll) { + tx_ = UringRing::create(send_options, nullptr, error, error_code); // its own submission thread + } + if (!tx_) { + rx_.reset(); + return false; + } + settings_.depth = rx_->sq_entries(); + if (settings_.fixed_files && (rx_->file_slots() == 0 || tx_->file_slots() == 0)) { + E3_LOG_WARN(LOG_TAG) << "The kernel refused registered files; using plain descriptors"; + settings_.fixed_files = false; + } + if (settings_.fixed_buffers && rx_->buffer_slots() == 0) { + E3_LOG_WARN(LOG_TAG) << "The kernel refused registered buffers; receiving into plain buffers"; + settings_.fixed_buffers = false; + } + slots_ = std::make_shared(); + slots_->rx = rx_.get(); + slots_->tx = tx_.get(); + slots_->files = settings_.fixed_files; + slots_->buffers = settings_.fixed_buffers; + if (slots_->files || slots_->buffers) { + for (int slot = static_cast(SLOT_COUNT) - 1; slot >= 0; --slot) slots_->free.push_back(slot); + } + wake_fd_ = eventfd(0, EFD_CLOEXEC); + if (wake_fd_ < 0) { + *error_code = errno; + *error = std::string("eventfd: ") + strerror(errno); + return false; + } + completions_.reserve(rx_->cq_entries()); + send_completions_.reserve(tx_->cq_entries()); + return true; +} + +UringE3Connector::~UringE3Connector() { + dispose(); +} + +SocketSpec UringE3Connector::spec() const { + return SocketSpec{transport_layer_, setup_endpoint_, parse_target(endpoint_uri_, port_), options_, LOG_TAG}; +} + +std::shared_ptr UringE3Connector::make_conn(int fd) { + auto conn = std::make_shared(); + conn->fd = fd; + conn->token = next_token_.fetch_add(1); + conn->transport = transport_layer_; + conn->slots = slots_; + conn->capacity = transport_layer_ == E3TransportLayer::TCP ? STREAM_CAPACITY : MESSAGE_CAPACITY; + conn->rx.reset(new uint8_t[conn->capacity]); + if (slots_->files || slots_->buffers) { + conn->slot = slots_->take(); + } + if (conn->slot >= 0) { + const auto slot = static_cast(conn->slot); + if (slots_->files) { + if (rx_->set_file(slot, fd) == 0 && tx_->set_file(slot, fd) == 0) { + conn->file_slot = conn->slot; + } else { + rx_->set_file(slot, -1); + tx_->set_file(slot, -1); + } + } + // SCTP needs recvmsg for the receive info, which takes no registered buffer. + if (slots_->buffers && transport_layer_ != E3TransportLayer::SCTP) { + const int rc = rx_->set_buffer(slot, conn->rx.get(), conn->capacity); + if (rc == 0) { + conn->buffer_slot = conn->slot; + } else { + E3_LOG_WARN(LOG_TAG) << "Could not register a receive buffer (" << strerror(-rc) + << "); this connection receives into a plain one"; + } + } + } + return conn; +} + +// =========================================================================== +// Reading: whoever holds reader_mutex_ drives the receive ring +// =========================================================================== + +template +bool UringE3Connector::prepare(UringRing& ring, Prepare&& how) { + // A full submission queue empties when it is handed over; with a kernel thread, when it has caught up. + for (int attempt = 0; attempt < 10000; ++attempt) { + if (how()) { + return true; + } + if (ring.submit() < 0 || attempt > 100) { + std::this_thread::yield(); + } + } + E3_LOG_ERROR(LOG_TAG) << "The submission queue stayed full"; + return false; +} + +bool UringE3Connector::arm_recv(Conn& conn) { + const uint64_t data = data_for(OP_RECV, conn.token); + bool armed; + if (transport_layer_ == E3TransportLayer::SCTP) { + conn.iov.iov_base = conn.rx.get() + conn.held; + conn.iov.iov_len = conn.capacity - conn.held; + conn.msg = {}; + conn.msg.msg_iov = &conn.iov; + conn.msg.msg_iovlen = 1; + conn.msg.msg_control = conn.control; + conn.msg.msg_controllen = sizeof(conn.control); + armed = prepare(*rx_, [&] { return rx_->recvmsg(conn.file(), &conn.msg, 0, data); }); + } else { + uint8_t* at = conn.rx.get() + conn.held; + const size_t room = conn.capacity - conn.held; + if (conn.buffer_slot >= 0) { + armed = prepare(*rx_, [&] { + return rx_->read_fixed(conn.file(), at, room, static_cast(conn.buffer_slot), data); + }); + } else { + // IPC: MSG_TRUNC makes the result the message's own length, so one too long is seen as such. + const int flags = transport_layer_ == E3TransportLayer::IPC ? MSG_TRUNC : 0; + armed = prepare(*rx_, [&] { return rx_->recv(conn.file(), at, room, flags, data); }); + } + } + conn.recv_in_flight = armed; + return armed; +} + +void UringE3Connector::arm() { + if (shutdown_requested_.load()) { + return; + } + if (!wake_armed_ && wake_fd_ >= 0) { + wake_armed_ = prepare(*rx_, [&] { return rx_->read_eventfd(wake_fd_, &wake_value_, data_for(OP_WAKE, 0)); }); + } + if (role_ == E3Role::DAPP) { + auto conn = client(); + if (conn && !conn->recv_in_flight && !conn->closed) { + arm_recv(*conn); + } + return; + } + if (listener_ >= 0 && !accept_armed_ && now_ns() >= accept_paused_until_ns_) { + accept_length_ = sizeof(accept_address_); + accept_armed_ = prepare(*rx_, [&] { + return rx_->accept(RingFile{listener_, -1}, reinterpret_cast(&accept_address_), + &accept_length_, data_for(OP_ACCEPT, 0)); + }); + } + for (auto& entry : conns_) { + Conn& conn = *entry.second; + if (!conn.recv_in_flight && !conn.closed) { + arm_recv(conn); + } + } +} + +/** + * @brief Turn what a completed receive brought into whole PDUs. + * + * TCP: stream bytes, cut by the length prefix. IPC: one receive is one PDU, and one longer than + * the limit is an error rather than a silent cut. SCTP: one recvmsg is one fragment of a message, + * joined up to MSG_EOR; notifications are skipped, and a payload protocol id other than 0 and + * libe3's is an error. + * @param transport The connector's transport + * @param rx The connection's receive buffer + * @param held Bytes kept in it from earlier receives; updated + * @param msg SCTP: the receive's header, with the flags and control data the kernel wrote + * @param first Nothing has been read from this connection yet + * @param result The completion's result: bytes, 0 for a close, or -errno + * @param pdus Whole PDUs are appended here + * @return MORE to go on receiving, CLOSED or FAILED to drop the connection + */ +static Got consume_into(E3TransportLayer transport, uint8_t* rx, size_t& held, const struct msghdr& msg, + bool first, int result, std::vector>& pdus) { + if (result == 0) { + return Got::CLOSED; + } + if (result < 0) { + if (result == -EAGAIN || result == -EINTR || result == -ECANCELED || result == -ENOBUFS) { + return Got::MORE; + } + E3_LOG_DEBUG(LOG_TAG) << "Receive failed: " << strerror(-result); + return Got::FAILED; + } + const auto n = static_cast(result); + if (transport == E3TransportLayer::TCP) { + held += n; + size_t offset = 0; + for (;;) { + size_t size = 0; + const StreamFrame frame = parse_stream_frame(rx + offset, held - offset, size); + if (frame == StreamFrame::NEED_MORE) { + break; + } + if (frame == StreamFrame::INVALID) { + E3_LOG_ERROR(LOG_TAG) << "Invalid PDU length " << size; + if (first && pdus.empty()) { + E3_LOG_INFO(LOG_TAG) << "The first message is not a length-prefixed PDU. A dApp built before " + "libe3 0.6 sends its setup request without a prefix."; + } + return Got::FAILED; + } + const uint8_t* start = rx + offset + TCP_PREFIX_BYTES; + pdus.emplace_back(start, start + size); + offset += TCP_PREFIX_BYTES + size; + } + if (offset > 0) { + std::memmove(rx, rx + offset, held - offset); + held -= offset; + } + return Got::MORE; + } + if (transport == E3TransportLayer::IPC) { + if (n > MAX_PDU) { + E3_LOG_ERROR(LOG_TAG) << "PDU longer than " << MAX_PDU << " bytes"; + return Got::FAILED; + } + pdus.emplace_back(rx, rx + n); + return Got::MORE; + } + if (sctp_is_notification(msg)) { + return Got::MORE; + } + if (held == 0 && !sctp_ppid_acceptable(msg, LOG_TAG)) { + return Got::FAILED; + } + held += n; + if (held > MAX_PDU) { + E3_LOG_ERROR(LOG_TAG) << "PDU longer than " << MAX_PDU << " bytes"; + return Got::FAILED; + } + if (msg.msg_flags & MSG_EOR) { + pdus.emplace_back(rx, rx + held); + held = 0; + } + return Got::MORE; +} + +void UringE3Connector::handle(const RingCompletion& completion) { + const uint64_t id = id_of(completion.user_data); + switch (op_of(completion.user_data)) { + case OP_WAKE: + wake_armed_ = false; + break; + case OP_ACCEPT: + accept_armed_ = false; + on_accept(completion.res); + break; + case OP_RECV: { + if (role_ == E3Role::DAPP) { + auto conn = client(); + if (!conn || conn->token != id) { + break; // a connection that was replaced + } + conn->recv_in_flight = false; + std::vector> pdus; + const Got got = consume_into(transport_layer_, conn->rx.get(), conn->held, conn->msg, + !conn->seen_first, completion.res, pdus); + for (auto& pdu : pdus) { + conn->seen_first = true; + client_ready_.push_back(std::move(pdu)); + } + if (got != Got::MORE) { + conn->closed = true; + (got == Got::CLOSED ? client_closed_ : client_failed_) = true; + } + break; + } + auto found = conns_.find(id); + if (found == conns_.end()) { + break; + } + std::shared_ptr conn = found->second; + conn->recv_in_flight = false; + on_recv(conn, completion.res); + break; + } + default: + break; + } +} + +void UringE3Connector::on_accept(int result) { + if (result < 0) { + if (result != -EINTR && result != -EAGAIN && result != -ECANCELED && result != -ECONNABORTED) { + E3_LOG_WARN(LOG_TAG) << "Accept failed: " << strerror(-result); + accept_paused_until_ns_ = now_ns() + 100'000'000; // a full descriptor table must not spin the reader + } + return; + } + const int fd = result; + size_t connected; + { + std::lock_guard lock(peers_mutex_); + connected = peers_.size(); + } + const SocketSpec where = spec(); + if (!admit(where, accept_address_, connected)) { + ::close(fd); + return; + } + tune_socket(where, fd, SocketKind::ACCEPTED); + auto conn = make_conn(fd); + conns_[conn->token] = conn; + size_t count; + { + std::lock_guard lock(peers_mutex_); + peers_.push_back(conn); + count = peers_.size(); + } + E3_LOG_INFO(LOG_TAG) << "Peer connected (" << count << " total)"; +} + +void UringE3Connector::on_recv(const std::shared_ptr& conn, int result) { + std::vector> pdus; + const Got got = consume_into(transport_layer_, conn->rx.get(), conn->held, conn->msg, !conn->seen_first, + result, pdus); + for (auto& pdu : pdus) { + deliver(conn, std::move(pdu)); + } + if (got != Got::MORE) { + drop(conn); + } +} + +void UringE3Connector::deliver(const std::shared_ptr& conn, std::vector&& pdu) { + ++produced_; + if (!conn->seen_first) { + conn->seen_first = true; + { + std::lock_guard lock(queue_mutex_); + setup_queue_.push_back(Ready{conn, conn->token, std::move(pdu), false}); + } + setup_ready_.notify_one(); + return; + } + std::lock_guard lock(queue_mutex_); + ready_.push_back(Ready{nullptr, conn->token, std::move(pdu), false}); +} + +void UringE3Connector::drop(const std::shared_ptr& conn) { + if (conn->closed) { + return; + } + conn->closed = true; + ++produced_; + ::shutdown(conn->fd, SHUT_RDWR); + conns_.erase(conn->token); + size_t remaining; + { + std::lock_guard lock(peers_mutex_); + peers_.erase(std::remove(peers_.begin(), peers_.end(), conn), peers_.end()); + remaining = peers_.size(); + } + { + // After the PDUs it sent, so the caller hears of the close only once they were taken. + std::lock_guard lock(queue_mutex_); + ready_.push_back(Ready{nullptr, conn->token, {}, true}); + } + if (!conn->seen_first) { + E3_LOG_INFO(LOG_TAG) << "A connection closed before it sent a complete setup request. A dApp built " + "before libe3 0.6 uses three connections and unframed messages, which " + "this endpoint does not serve."; + } + E3_LOG_INFO(LOG_TAG) << "Peer disconnected (" << remaining << " remain)"; +} + +void UringE3Connector::pump(int timeout_ms) { + produced_ = 0; + const int64_t deadline = now_ns() + static_cast(timeout_ms) * 1'000'000; + for (;;) { + arm(); + const int64_t left_us = (deadline - now_ns()) / 1000; + if (left_us <= 0) { + return; + } + completions_.clear(); + const int got = rx_->wait(completions_, 1, left_us, settings_.busy_poll_us); + if (got < 0) { + E3_LOG_ERROR(LOG_TAG) << "Waiting on the receive ring failed: " << strerror(-got); + return; + } + for (const auto& completion : completions_) { + handle(completion); + } + if (produced_ > 0 || shutdown_requested_.load()) { + return; + } + } +} + +bool UringE3Connector::take_ready(std::vector& buffer, bool& closed_noted) { + std::lock_guard lock(queue_mutex_); + while (!ready_.empty() && ready_.front().closed) { + { + std::lock_guard closed(closed_mutex_); + closed_peers_.push_back(ready_.front().token); + } + ready_.pop_front(); + closed_noted = true; + } + if (ready_.empty()) { + return false; + } + Ready& next = ready_.front(); + buffer = std::move(next.data); + last_peer_token_ = next.token; + ready_.pop_front(); + return true; +} + +bool UringE3Connector::receiver_idle() const { + return now_ns() - last_receive_ns_.load() > SETUP_PUMP_AFTER_NS; +} + +/** + * Take back every operation armed on the receive ring, and handle what completes meanwhile as + * usual (data that arrived is kept). The kernel runs a completion's work on the thread that armed + * the operation, and the signal it uses to make that thread run it cuts short that thread's other + * blocking calls (a connect() with a send timeout returns EINTR). So a thread that reads only in + * passing (the setup thread standing in for an idle receive()) leaves nothing armed behind it. + */ +void UringE3Connector::drain_rx_locked(int64_t timeout_us) { + if (!rx_) { + return; + } + auto armed = [&] { + std::vector found; + if (accept_armed_) found.push_back(data_for(OP_ACCEPT, 0)); + if (wake_armed_) found.push_back(data_for(OP_WAKE, 0)); + for (auto& entry : conns_) { + if (entry.second->recv_in_flight) found.push_back(data_for(OP_RECV, entry.first)); + } + if (auto conn = client(); conn && conn->recv_in_flight) { + found.push_back(data_for(OP_RECV, conn->token)); + } + return found; + }; + std::vector in_flight = armed(); + if (in_flight.empty()) { + return; + } + for (uint64_t data : in_flight) { + prepare(*rx_, [&] { return rx_->cancel(data, data_for(OP_CANCEL, 0)); }); + } + const int64_t deadline = now_ns() + timeout_us * 1000; + while (!armed().empty() && now_ns() < deadline) { + completions_.clear(); + if (rx_->wait(completions_, 1, std::max(1, (deadline - now_ns()) / 1000), 0) < 0) { + break; + } + for (const auto& completion : completions_) { + handle(completion); + } + } +} + +// =========================================================================== +// RAN role +// =========================================================================== + +ErrorCode UringE3Connector::setup_initial_connection() { + // A blocking listener: io_uring gives -EAGAIN back for an O_NONBLOCK one instead of waiting. + listener_ = listen_on(spec(), /*nonblocking=*/false); + if (listener_ < 0) { + return ErrorCode::CONNECTION_FAILED; + } + connected_ = true; + E3_LOG_INFO(LOG_TAG) << "Listening on " << setup_endpoint_ << " port " << port_; + return ErrorCode::SUCCESS; +} + +ErrorCode UringE3Connector::setup_inbound_connection() { + return listener_ >= 0 ? ErrorCode::SUCCESS : ErrorCode::NOT_INITIALIZED; +} + +ErrorCode UringE3Connector::setup_outbound_connection() { + return listener_ >= 0 ? ErrorCode::SUCCESS : ErrorCode::NOT_INITIALIZED; +} + +int UringE3Connector::receive(std::vector& buffer) { + if (role_ == E3Role::DAPP) { + auto conn = client(); + if (!conn) { + return static_cast(ErrorCode::NOT_CONNECTED); + } + if (awaiting_setup_response_.load()) { + std::unique_lock lock(await_mutex_); + await_cv_.wait_for(lock, std::chrono::milliseconds(RECEIVE_TIMEOUT_MS), [this] { + return !awaiting_setup_response_.load() || shutdown_requested_.load(); + }); + if (awaiting_setup_response_.load()) { + return 0; + } + } + std::lock_guard reader(reader_mutex_); + return read_client(buffer, RECEIVE_TIMEOUT_MS); + } + if (listener_ < 0) { + return static_cast(ErrorCode::NOT_CONNECTED); + } + last_receive_ns_.store(now_ns()); + bool closed_noted = false; + if (take_ready(buffer, closed_noted)) { + return static_cast(buffer.size()); + } + if (closed_noted) { + return 0; // so the caller learns of the closed peers now rather than after the next wait + } + { + std::lock_guard reader(reader_mutex_); + if (!shutdown_requested_.load()) { + pump(RECEIVE_TIMEOUT_MS); + } + } + if (take_ready(buffer, closed_noted)) { + return static_cast(buffer.size()); + } + return 0; +} + +int UringE3Connector::recv_setup_request(std::vector& buffer) { + if (listener_ < 0) { + return static_cast(ErrorCode::NOT_CONNECTED); + } + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(RECEIVE_TIMEOUT_MS); + for (;;) { + { + std::unique_lock lock(queue_mutex_); + if (!setup_queue_.empty()) { + Ready item = std::move(setup_queue_.front()); + setup_queue_.pop_front(); + buffer = std::move(item.data); + setup_conn_ = item.conn; + last_setup_token_ = item.token; + return static_cast(buffer.size()); + } + } + if (shutdown_requested_.load() || std::chrono::steady_clock::now() >= deadline) { + return 0; + } + // The inbound thread is the reader. When nothing calls receive() (a caller that only + // serves setup), this thread reads instead. + if (receiver_idle() && reader_mutex_.try_lock()) { + pump(SETUP_PUMP_MS); + drain_rx_locked(100'000); + reader_mutex_.unlock(); + } else { + std::unique_lock lock(queue_mutex_); + setup_ready_.wait_for(lock, std::chrono::milliseconds(50), + [this] { return !setup_queue_.empty() || shutdown_requested_.load(); }); + } + } +} + +ErrorCode UringE3Connector::send_response(const std::vector& data) { + std::shared_ptr conn = setup_conn_; + if (!conn) { + return ErrorCode::NOT_CONNECTED; + } + if (shutdown_requested_.load()) { + return ErrorCode::CANCELLED; + } + std::lock_guard lock(tx_mutex_); + targets_.assign(1, conn); + return transmit(data, PduClass::PROCEDURE); +} + +ErrorCode UringE3Connector::send(const std::vector& data) { + return send_to(0, data, PduClass::PROCEDURE); +} + +ErrorCode UringE3Connector::send_to(uint64_t peer_token, const std::vector& data, PduClass kind) { + // Checked first: shutdown() leaves the sockets open until dispose(), so a send racing it + // would otherwise be a real EPIPE, indistinguishable from a transport failure. + if (shutdown_requested_.load()) { + return ErrorCode::CANCELLED; + } + if (data.empty() || data.size() > MAX_PDU) { + E3_LOG_ERROR(LOG_TAG) << "PDU of " << data.size() << " bytes cannot be sent"; + return ErrorCode::INVALID_PARAM; + } + if (role_ == E3Role::DAPP) { + auto conn = client(); + if (!conn) { + return ErrorCode::NOT_CONNECTED; + } + std::lock_guard lock(tx_mutex_); + targets_.assign(1, conn); + return transmit(data, PduClass::PROCEDURE); + } + if (listener_ < 0) { + return ErrorCode::NOT_CONNECTED; + } + std::lock_guard lock(tx_mutex_); + targets_.clear(); + { + std::lock_guard peers(peers_mutex_); + if (peer_token == 0) { + targets_.assign(peers_.begin(), peers_.end()); + } else { + for (const auto& conn : peers_) { + if (conn->token == peer_token) { + targets_.push_back(conn); + break; + } + } + } + } + if (targets_.empty()) { + return peer_token == 0 ? ErrorCode::SUCCESS : ErrorCode::NOT_CONNECTED; + } + return transmit(data, kind); +} + +// =========================================================================== +// Sending: whoever holds tx_mutex_ drives the send ring +// =========================================================================== + +bool UringE3Connector::prepare_send(size_t index, const std::vector& data, PduClass kind) { + SendOp& op = send_ops_[index]; + const RingFile file = op.conn->file(); + const uint64_t user_data = data_for(OP_SEND, index); + op.msg = {}; + if (transport_layer_ == E3TransportLayer::TCP) { + // The prefix and the PDU in one write; after a short write, what is left of them. + size_t count = 0; + if (op.done < TCP_PREFIX_BYTES) { + op.iov[count++] = iovec{prefix_ + op.done, TCP_PREFIX_BYTES - op.done}; + op.iov[count++] = iovec{const_cast(data.data()), data.size()}; + } else { + const size_t skip = op.done - TCP_PREFIX_BYTES; + op.iov[count++] = iovec{const_cast(data.data()) + skip, data.size() - skip}; + } + op.msg.msg_iov = op.iov; + op.msg.msg_iovlen = count; + return prepare(*tx_, [&] { return tx_->sendmsg(file, &op.msg, MSG_NOSIGNAL, user_data); }); + } + if (transport_layer_ == E3TransportLayer::IPC) { + return prepare(*tx_, [&] { return tx_->send(file, data.data(), data.size(), MSG_NOSIGNAL, user_data); }); + } + op.iov[0] = iovec{const_cast(data.data()), data.size()}; + op.msg.msg_iov = op.iov; + op.msg.msg_iovlen = 1; + set_sctp_send_info(op.msg, op.control, kind); + return prepare(*tx_, [&] { return tx_->sendmsg(file, &op.msg, MSG_NOSIGNAL, user_data); }); +} + +/** + * Send @p data to every connection in targets_, all writes submitted together, and wait until + * each is done or SEND_TIMEOUT_MS has passed. A write still waiting then is cancelled and its + * connection shut, as the POSIX link drops a peer whose send timed out; the reader then reports + * it closed. Returns only when the kernel holds no reference to the data any more. + */ +ErrorCode UringE3Connector::transmit(const std::vector& data, PduClass kind) { + const size_t count = targets_.size(); + if (send_ops_.size() < count) { + send_ops_.resize(count); + } + const size_t total = (transport_layer_ == E3TransportLayer::TCP ? TCP_PREFIX_BYTES : 0) + data.size(); + if (transport_layer_ == E3TransportLayer::TCP) { + write_stream_prefix(prefix_, data.size()); + } + size_t outstanding = 0; + for (size_t i = 0; i < count; ++i) { + SendOp& op = send_ops_[i]; + op.conn = targets_[i]; + op.done = 0; + op.error = 0; + op.state = 0; + if (prepare_send(i, data, kind)) { + ++outstanding; + } else { + op.state = 2; + op.error = EBUSY; + } + } + + auto complete = [&](const RingCompletion& c, bool after_deadline) { + if (op_of(c.user_data) != OP_SEND) { + return; + } + SendOp& op = send_ops_[id_of(c.user_data)]; + if (op.state != 0) { + return; + } + if (c.res >= 0) { + op.done += static_cast(c.res); + if (op.done >= total) { + op.state = 1; + } else if (transport_layer_ == E3TransportLayer::TCP && !after_deadline) { + if (prepare_send(id_of(c.user_data), data, kind)) return; // the rest of a short write + op.state = 2; + op.error = EBUSY; + } else { + op.state = 2; + op.error = after_deadline ? ETIMEDOUT : EMSGSIZE; + } + } else if ((c.res == -EINTR || c.res == -EAGAIN) && !after_deadline) { + if (prepare_send(id_of(c.user_data), data, kind)) return; + op.state = 2; + op.error = EBUSY; + } else { + op.state = 2; + op.error = c.res == -ECANCELED ? ETIMEDOUT : -c.res; + } + --outstanding; + }; + + const int64_t deadline = now_ns() + static_cast(SEND_TIMEOUT_MS) * 1'000'000; + while (outstanding > 0) { + const int64_t left_us = (deadline - now_ns()) / 1000; + if (left_us <= 0) { + break; + } + send_completions_.clear(); + const int got = tx_->wait(send_completions_, 1, left_us, settings_.busy_poll_us); + if (got < 0) { + E3_LOG_ERROR(LOG_TAG) << "Waiting on the send ring failed: " << strerror(-got); + break; + } + for (const auto& c : send_completions_) { + complete(c, false); + } + } + // Writes still waiting: cancel them, and wait until the kernel lets go of the buffers. + while (outstanding > 0) { + for (size_t i = 0; i < count; ++i) { + if (send_ops_[i].state == 0) { + const uint64_t target = data_for(OP_SEND, i); + prepare(*tx_, [&] { return tx_->cancel(target, data_for(OP_CANCEL, i)); }); + } + } + send_completions_.clear(); + tx_->wait(send_completions_, 1, 100'000, 0); + for (const auto& c : send_completions_) { + complete(c, true); + } + } + + size_t sent = 0; + size_t failed = 0; + for (size_t i = 0; i < count; ++i) { + SendOp& op = send_ops_[i]; + if (op.state == 1) { + ++sent; + } else { + ++failed; + E3_LOG_ERROR(LOG_TAG) << "Failed to send: " << strerror(op.error); + // Dead or stalled: shutting it makes the reader drop it and report it closed. + ::shutdown(op.conn->fd, SHUT_RDWR); + } + op.conn.reset(); + } + targets_.clear(); + if (failed > 0 && sent == 0) { + return ErrorCode::TRANSPORT_ERROR; + } + return ErrorCode::SUCCESS; +} + +uint64_t UringE3Connector::last_peer_token() const { + return last_peer_token_; +} + +uint64_t UringE3Connector::last_setup_peer_token() const { + return last_setup_token_; +} + +std::vector UringE3Connector::take_closed_peers() { + std::lock_guard lock(closed_mutex_); + std::vector closed; + closed.swap(closed_peers_); + return closed; +} + +// =========================================================================== +// dApp role: one connection +// =========================================================================== + +std::shared_ptr UringE3Connector::client() const { + std::lock_guard lock(client_mutex_); + return client_; +} + +ErrorCode UringE3Connector::connect_client() { + const SocketSpec where = spec(); + int fd = -1; + for (int attempt = 0; attempt < CONNECT_ATTEMPTS && fd < 0; ++attempt) { + fd = connect_to(where); + if (fd < 0) { + if (shutdown_requested_.load()) { + return ErrorCode::CANCELLED; + } + std::this_thread::sleep_for(std::chrono::milliseconds(CONNECT_PAUSE_MS)); + } + } + if (fd < 0) { + E3_LOG_ERROR(LOG_TAG) << "Failed to connect to " << setup_endpoint_ << " port " << where.target.port + << ": " << strerror(errno); + return ErrorCode::CONNECTION_FAILED; + } + auto conn = make_conn(fd); + { + std::lock_guard lock(client_mutex_); + client_ = conn; + } + client_ready_.clear(); + client_closed_ = false; + client_failed_ = false; + // Until the setup response is read, the setup thread is the only reader: the inbound + // thread must not take it for an indication. + awaiting_setup_response_.store(true); + connected_ = true; + E3_LOG_INFO(LOG_TAG) << "Connected to " << setup_endpoint_ << " port " << where.target.port; + return ErrorCode::SUCCESS; +} + +void UringE3Connector::close_client_locked() { + auto conn = client(); + if (!conn) { + return; + } + ::shutdown(conn->fd, SHUT_RDWR); + // Its receive comes back with the shutdown; until then the kernel may still write into its buffer. + const int64_t deadline = now_ns() + 1'000'000'000; + bool cancelled = false; + while (conn->recv_in_flight && now_ns() < deadline) { + if (!cancelled && now_ns() > deadline - 900'000'000) { + const uint64_t target = data_for(OP_RECV, conn->token); + prepare(*rx_, [&] { return rx_->cancel(target, data_for(OP_CANCEL, 0)); }); + cancelled = true; + } + completions_.clear(); + rx_->wait(completions_, 1, 100'000, 0); + for (const auto& c : completions_) { + if (op_of(c.user_data) == OP_RECV && id_of(c.user_data) == conn->token) { + conn->recv_in_flight = false; + } else if (op_of(c.user_data) == OP_WAKE) { + wake_armed_ = false; + } + } + } + if (conn->recv_in_flight) { + retired_.push_back(conn); // kept until the rings are gone + } + { + std::lock_guard lock(client_mutex_); + client_.reset(); + } + client_ready_.clear(); +} + +ErrorCode UringE3Connector::setup_initial_connection_client() { + return connect_client(); +} + +ErrorCode UringE3Connector::send_setup_request_client(const std::vector& data) { + if (!client()) { + return ErrorCode::NOT_CONNECTED; + } + // The RAN takes the first PDU of a connection for the setup request, so a second + // attempt needs a connection of its own, as a ZeroMQ request socket is reopened. + if (setup_request_sent_) { + { + std::lock_guard reader(reader_mutex_); + close_client_locked(); + } + ErrorCode rc = connect_client(); + if (rc != ErrorCode::SUCCESS) { + return rc; + } + } + setup_request_sent_ = true; + return send_to(0, data, PduClass::PROCEDURE); +} + +int UringE3Connector::recv_setup_response_client(std::vector& buffer) { + if (!client()) { + return static_cast(ErrorCode::NOT_CONNECTED); + } + if (!awaiting_setup_response_.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + return 0; + } + std::lock_guard reader(reader_mutex_); + const int n = read_client(buffer, RECEIVE_TIMEOUT_MS); + if (n > 0) { + awaiting_setup_response_.store(false); + await_cv_.notify_all(); + } + return n; +} + +int UringE3Connector::read_client(std::vector& buffer, int timeout_ms) { + if (!client() || !rx_) { + return static_cast(ErrorCode::NOT_CONNECTED); + } + const int64_t deadline = now_ns() + static_cast(timeout_ms) * 1'000'000; + for (;;) { + if (!client_ready_.empty()) { + buffer = std::move(client_ready_.front()); + client_ready_.pop_front(); + return static_cast(buffer.size()); + } + if (client_failed_) { + return static_cast(ErrorCode::TRANSPORT_ERROR); + } + if (client_closed_) { + return static_cast(ErrorCode::NOT_CONNECTED); + } + if (shutdown_requested_.load()) { + return 0; + } + arm(); + const int64_t left_us = (deadline - now_ns()) / 1000; + if (left_us <= 0) { + return 0; + } + completions_.clear(); + const int got = rx_->wait(completions_, 1, left_us, settings_.busy_poll_us); + if (got < 0) { + E3_LOG_ERROR(LOG_TAG) << "Waiting on the receive ring failed: " << strerror(-got); + return static_cast(ErrorCode::TRANSPORT_ERROR); + } + for (const auto& completion : completions_) { + handle(completion); + } + } +} + +ErrorCode UringE3Connector::setup_inbound_connection_client() { + return client() ? ErrorCode::SUCCESS : ErrorCode::NOT_CONNECTED; +} + +ErrorCode UringE3Connector::setup_outbound_connection_client() { + return client() ? ErrorCode::SUCCESS : ErrorCode::NOT_CONNECTED; +} + +// =========================================================================== +// Shutdown +// =========================================================================== + +void UringE3Connector::shutdown() { + E3_LOG_DEBUG(LOG_TAG) << "Shutting down io_uring connector"; + shutdown_requested_.store(true); + setup_ready_.notify_all(); + await_cv_.notify_all(); + if (listener_ >= 0) { + ::shutdown(listener_, SHUT_RDWR); + } + { + std::lock_guard lock(peers_mutex_); + for (const auto& conn : peers_) { + ::shutdown(conn->fd, SHUT_RDWR); + } + } + if (auto conn = client()) { + ::shutdown(conn->fd, SHUT_RDWR); + } + if (wake_fd_ >= 0) { + const uint64_t one = 1; + ssize_t written = ::write(wake_fd_, &one, sizeof(one)); + (void)written; + } +} + +void UringE3Connector::dispose() { + if (disposed_) { + return; + } + disposed_ = true; + E3_LOG_DEBUG(LOG_TAG) << "Disposing io_uring connector"; + { + std::lock_guard lock(peers_mutex_); + for (const auto& conn : peers_) { + ::shutdown(conn->fd, SHUT_RDWR); + } + } + if (auto conn = client()) { + ::shutdown(conn->fd, SHUT_RDWR); + } + if (wake_fd_ >= 0) { + const uint64_t one = 1; + ssize_t written = ::write(wake_fd_, &one, sizeof(one)); + (void)written; + } + { + std::lock_guard reader(reader_mutex_); + drain_rx_locked(200'000); + conns_.clear(); + client_ready_.clear(); + } + { + std::lock_guard lock(queue_mutex_); + setup_queue_.clear(); + ready_.clear(); + } + setup_conn_.reset(); + { + std::lock_guard lock(peers_mutex_); + peers_.clear(); + } + { + std::lock_guard lock(client_mutex_); + client_.reset(); + } + { + std::lock_guard lock(closed_mutex_); + closed_peers_.clear(); + } + { + std::lock_guard lock(tx_mutex_); + targets_.clear(); + send_ops_.clear(); + } + if (slots_) { + std::lock_guard lock(slots_->mu); + slots_->rx = nullptr; + slots_->tx = nullptr; + } + tx_.reset(); + rx_.reset(); + retired_.clear(); + if (listener_ >= 0) { + ::close(listener_); + listener_ = -1; + // Only the RAN, which binds, has a socket file to remove. The directory is shared + // between agents and tests and stays, so that removing it cannot race a bind in it. + if (transport_layer_ == E3TransportLayer::IPC) { + unlink(setup_endpoint_.c_str()); + } + } + if (wake_fd_ >= 0) { + ::close(wake_fd_); + wake_fd_ = -1; + } + connected_ = false; + E3_LOG_INFO(LOG_TAG) << "io_uring connector disposed"; +} + +} // namespace libe3 diff --git a/src/connector/uring_connector.hpp b/src/connector/uring_connector.hpp new file mode 100644 index 00000000..36a79283 --- /dev/null +++ b/src/connector/uring_connector.hpp @@ -0,0 +1,231 @@ +/** + * @file uring_connector.hpp + * @brief The io_uring E3 connector: the POSIX link's wire, with the waiting and the sends done through io_uring + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef LIBE3_URING_CONNECTOR_HPP +#define LIBE3_URING_CONNECTOR_HPP + +#include "libe3/e3_connector.hpp" +#include "socket_link.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace libe3 { + +class UringRing; +struct RingCompletion; + +/** + * @brief io_uring E3 connector: same endpoint, sockets and bytes as PosixE3Connector + * + * A peer cannot tell this link from the POSIX one: one connection per dApp on one endpoint, + * SCTP, TCP or IPC (a sequenced-packet socket), one PDU per message, TCP length-prefixed both + * ways, SCTP payload protocol id 0x00E3A001 with indications on stream 1. A RAN of one link + * serves a dApp of the other. What differs is how the bytes move: + * + * - Two rings. The receive ring holds a receive armed on every connection (and the accept, + * on a RAN); whichever thread reads waits on it, so a received PDU costs one io_uring_enter + * (re-arm and wait together) where POSIX needs poll() and recv(). The send ring takes the + * writes; a RAN that sends to N dApps submits the N writes at once and waits for them + * together, so a peer that stopped reading delays no other peer. + * - No thread of its own. The threads that call receive() and send() drive the rings, so + * send() returns when the kernel has the bytes, as with POSIX, and SEND_DONE means the same + * on both links. + * - Ring options, read once from the environment when the connector is made, none on by + * default except registered files: LIBE3_URING_DEPTH, LIBE3_URING_FIXED_FILES, + * LIBE3_URING_FIXED_BUFFERS, LIBE3_URING_SQPOLL, LIBE3_URING_SQPOLL_CPU and + * LIBE3_URING_BUSY_POLL_US (docs/connectors/uring.md). Each falls back to the plain way + * when the kernel refuses it; settings() says what is in effect. + * + * Linux 5.11 or later with io_uring allowed for the process (Docker's default seccomp + * profile and kernel.io_uring_disabled refuse it): create() returns null and logs why otherwise. + */ +class UringE3Connector : public E3Connector { +public: + /** @brief The ring options, as asked for or, from settings(), as in effect. */ + struct Settings { + unsigned depth{64}; ///< LIBE3_URING_DEPTH: submission queue entries of each ring + bool fixed_files{true}; ///< LIBE3_URING_FIXED_FILES: sockets in the rings' registered-file tables + bool fixed_buffers{false}; ///< LIBE3_URING_FIXED_BUFFERS: receive into registered buffers (TCP, IPC) + bool sqpoll{false}; ///< LIBE3_URING_SQPOLL: a kernel thread takes the submissions + int sqpoll_cpu{-1}; ///< LIBE3_URING_SQPOLL_CPU: the CPU of that thread + unsigned busy_poll_us{0}; ///< LIBE3_URING_BUSY_POLL_US: poll for a completion this long before sleeping + + /** + * @brief The options set in the environment, the defaults for the others. + * @return The settings asked for + */ + static Settings from_environment(); + /** + * @brief One line for a log. + * @return The settings in words + */ + std::string describe() const; + }; + + /** + * @brief Make a connector, or null when io_uring cannot be used here (the reason is logged) + * @param transport_layer SCTP, TCP or IPC + * @param endpoint As for PosixE3Connector + * @param port As for PosixE3Connector + * @param options Socket and admission options, as for PosixE3Connector + * @return The connector, or null + */ + static std::unique_ptr create( + E3TransportLayer transport_layer, + const std::string& endpoint, + uint16_t port, + const PosixOptions& options + ); + + /** @brief Close every socket and both rings. */ + ~UringE3Connector() override; + + ErrorCode setup_initial_connection() override; + int recv_setup_request(std::vector& buffer) override; + ErrorCode send_response(const std::vector& data) override; + ErrorCode setup_inbound_connection() override; + int receive(std::vector& buffer) override; + ErrorCode setup_outbound_connection() override; + ErrorCode send(const std::vector& data) override; + ErrorCode send_to(uint64_t peer_token, const std::vector& data, PduClass kind) override; + + ErrorCode setup_initial_connection_client() override; + ErrorCode send_setup_request_client(const std::vector& data) override; + int recv_setup_response_client(std::vector& buffer) override; + ErrorCode setup_inbound_connection_client() override; + ErrorCode setup_outbound_connection_client() override; + + void dispose() override; + void shutdown() override; + uint64_t last_peer_token() const override; + uint64_t last_setup_peer_token() const override; + std::vector take_closed_peers() override; + + E3LinkLayer link_layer() const noexcept override { + return E3LinkLayer::URING; + } + + E3TransportLayer transport_layer() const noexcept override { + return transport_layer_; + } + + bool is_connected() const noexcept override { + return connected_; + } + + /** + * @brief The ring options in effect: those asked for, less what the kernel refused. + * @return The settings in effect + */ + const Settings& settings() const noexcept { + return settings_; + } + +private: + struct Conn; + struct Slots; + struct Ready; + struct SendOp; + + UringE3Connector(E3TransportLayer transport_layer, const std::string& endpoint, uint16_t port, + const PosixOptions& options, const Settings& settings); + bool open(std::string* error, int* error_code); + socket_link::SocketSpec spec() const; + + std::shared_ptr make_conn(int fd); + void arm(); + bool arm_recv(Conn& conn); + template bool prepare(UringRing& ring, Prepare&& how); + void pump(int timeout_ms); + void handle(const RingCompletion& completion); + void on_accept(int result); + void on_recv(const std::shared_ptr& conn, int result); + void deliver(const std::shared_ptr& conn, std::vector&& pdu); + void drop(const std::shared_ptr& conn); + bool take_ready(std::vector& buffer, bool& closed_noted); + bool receiver_idle() const; + ErrorCode transmit(const std::vector& data, PduClass kind); + bool prepare_send(size_t index, const std::vector& data, PduClass kind); + int read_client(std::vector& buffer, int timeout_ms); + void close_client_locked(); + ErrorCode connect_client(); + std::shared_ptr client() const; + void drain_rx_locked(int64_t timeout_us); + + E3TransportLayer transport_layer_; + std::string endpoint_uri_; + uint16_t port_; + PosixOptions options_; + Settings settings_; + + std::unique_ptr rx_; ///< Accepts and receives; driven by whoever holds reader_mutex_ + std::unique_ptr tx_; ///< Sends; driven by whoever holds tx_mutex_ + std::shared_ptr slots_; ///< Registered-file and buffer slots, shared with the connections + int wake_fd_{-1}; ///< An eventfd that shutdown() writes to wake the reader + + // Reader state: only the holder of reader_mutex_ touches it. + std::mutex reader_mutex_; + std::unordered_map> conns_; ///< RAN role: by token + std::vector completions_; + sockaddr_storage accept_address_{}; + socklen_t accept_length_{0}; + bool accept_armed_{false}; + int64_t accept_paused_until_ns_{0}; + uint64_t wake_value_{0}; + bool wake_armed_{false}; + size_t produced_{0}; ///< PDUs and closes the current pump handed on + std::deque> client_ready_; ///< dApp role: PDUs read and not yet returned + bool client_closed_{false}; + bool client_failed_{false}; + + int listener_{-1}; + std::atomic next_token_{1}; ///< Never reused by this connector + mutable std::mutex peers_mutex_; ///< Guards peers_ + std::vector> peers_; ///< RAN role: what senders may send to + std::atomic last_receive_ns_{0}; + + std::mutex queue_mutex_; ///< Guards setup_queue_ and ready_ + std::condition_variable setup_ready_; + std::deque setup_queue_; ///< First PDUs of connections + std::deque ready_; ///< Later PDUs, and closes, in the order they came + std::shared_ptr setup_conn_; ///< Sender of the last setup request, for send_response + uint64_t last_setup_token_{0}; + uint64_t last_peer_token_{0}; + std::mutex closed_mutex_; + std::vector closed_peers_; + + std::mutex tx_mutex_; ///< Held by whoever drives the send ring + std::vector> targets_; ///< Where the current send goes + std::vector send_ops_; + std::vector send_completions_; + uint8_t prefix_[socket_link::TCP_PREFIX_BYTES]{}; + + mutable std::mutex client_mutex_; ///< Guards client_ + std::shared_ptr client_; ///< dApp role: the one connection + std::vector> retired_; ///< Replaced connections the kernel had not let go of + bool setup_request_sent_{false}; + std::atomic awaiting_setup_response_{false}; + std::mutex await_mutex_; + std::condition_variable await_cv_; + + bool connected_{false}; + bool disposed_{false}; + std::atomic shutdown_requested_{false}; +}; + +} // namespace libe3 + +#endif // LIBE3_URING_CONNECTOR_HPP diff --git a/src/connector/uring_ring.cpp b/src/connector/uring_ring.cpp new file mode 100644 index 00000000..11aadfc2 --- /dev/null +++ b/src/connector/uring_ring.cpp @@ -0,0 +1,287 @@ +/** + * @file uring_ring.cpp + * @brief One io_uring instance: the only code of libe3 that includes + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "uring_ring.hpp" + +#include +#include +#include +#include +#include +#include + +namespace libe3 { + +namespace { + +int64_t now_us() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +inline void cpu_relax() { +#if defined(__x86_64__) || defined(__i386__) + __builtin_ia32_pause(); +#elif defined(__aarch64__) || defined(__arm__) + asm volatile("yield" ::: "memory"); +#endif +} + +/// What the connector needs. All are older than IORING_FEAT_EXT_ARG (5.11), which is required too. +constexpr int REQUIRED_OPS[] = { + IORING_OP_ACCEPT, IORING_OP_RECV, IORING_OP_SEND, IORING_OP_RECVMSG, IORING_OP_SENDMSG, + IORING_OP_READ, IORING_OP_ASYNC_CANCEL, IORING_OP_READ_FIXED, +}; + +} // namespace + +/** @brief The liburing state, kept out of the header. */ +struct UringRing::State { + struct io_uring ring{}; ///< The ring + bool initialised{false}; ///< io_uring_queue_init_params succeeded, so exit is due +}; + +UringRing::UringRing() : state_(std::make_unique()) {} + +UringRing::~UringRing() { + if (state_ && state_->initialised) { + io_uring_queue_exit(&state_->ring); + } +} + +std::unique_ptr UringRing::create(const RingOptions& options, const UringRing* share_kernel_thread_with, + std::string* error, int* error_code) { + std::unique_ptr self(new UringRing()); + struct io_uring_params params{}; + params.flags = IORING_SETUP_CLAMP | IORING_SETUP_CQSIZE; + params.cq_entries = options.depth * 4; + if (options.sqpoll) { + params.flags |= IORING_SETUP_SQPOLL; + params.sq_thread_idle = options.sqpoll_idle_ms; + if (options.sqpoll_cpu >= 0) { + params.flags |= IORING_SETUP_SQ_AFF; + params.sq_thread_cpu = static_cast(options.sqpoll_cpu); + } + if (share_kernel_thread_with != nullptr && share_kernel_thread_with->sqpoll_) { + params.flags |= IORING_SETUP_ATTACH_WQ; + params.wq_fd = static_cast(share_kernel_thread_with->state_->ring.ring_fd); + } + } + int rc = io_uring_queue_init_params(options.depth, &self->state_->ring, ¶ms); + if (rc < 0) { + *error_code = -rc; + *error = std::string("io_uring_queue_init_params: ") + strerror(-rc); + return nullptr; + } + self->state_->initialised = true; + self->sqpoll_ = options.sqpoll; + + if (!(params.features & IORING_FEAT_EXT_ARG) || !(params.features & IORING_FEAT_FAST_POLL)) { + *error_code = ENOSYS; + *error = "this kernel's io_uring lacks IORING_FEAT_EXT_ARG or IORING_FEAT_FAST_POLL (Linux 5.11 or later is needed)"; + return nullptr; + } + struct io_uring_probe* probe = io_uring_get_probe_ring(&self->state_->ring); + if (probe == nullptr) { + *error_code = ENOSYS; + *error = "this kernel cannot say which io_uring operations it has (Linux 5.11 or later is needed)"; + return nullptr; + } + for (int op : REQUIRED_OPS) { + if (!io_uring_opcode_supported(probe, op)) { + io_uring_free_probe(probe); + *error_code = ENOSYS; + *error = "this kernel's io_uring lacks operation " + std::to_string(op); + return nullptr; + } + } + io_uring_free_probe(probe); + + // Optional: a kernel that refuses them leaves the ring working on plain descriptors and buffers. + if (options.file_slots > 0) { + rc = io_uring_register_files_sparse(&self->state_->ring, options.file_slots); + if (rc < 0) { + std::vector empty(options.file_slots, -1); + rc = io_uring_register_files(&self->state_->ring, empty.data(), options.file_slots); + } + if (rc == 0) { + self->file_slots_ = options.file_slots; + } + } + if (options.buffer_slots > 0 + && io_uring_register_buffers_sparse(&self->state_->ring, options.buffer_slots) == 0) { + self->buffer_slots_ = options.buffer_slots; + } + return self; +} + +unsigned UringRing::sq_entries() const noexcept { return state_->ring.sq.ring_entries; } +unsigned UringRing::cq_entries() const noexcept { return state_->ring.cq.ring_entries; } + +int UringRing::set_file(unsigned slot, int fd) { + const int rc = io_uring_register_files_update(&state_->ring, slot, &fd, 1); + return rc < 0 ? rc : 0; +} + +int UringRing::set_buffer(unsigned slot, void* base, size_t len) { + struct iovec iov{base, base != nullptr ? len : 0}; + __u64 tag = 0; + const int rc = io_uring_register_buffers_update_tag(&state_->ring, slot, &iov, &tag, 1); + return rc < 0 ? rc : 0; +} + +namespace { + +inline struct io_uring_sqe* next_sqe(struct io_uring& ring) { + return io_uring_get_sqe(&ring); +} + +inline void finish(struct io_uring_sqe* sqe, RingFile file, uint64_t user_data) { + if (file.slot >= 0) { + sqe->flags |= IOSQE_FIXED_FILE; + } + io_uring_sqe_set_data64(sqe, user_data); +} + +inline int target(RingFile file) { return file.slot >= 0 ? file.slot : file.fd; } + +} // namespace + +bool UringRing::accept(RingFile file, sockaddr* address, socklen_t* length, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_accept(sqe, target(file), address, length, SOCK_CLOEXEC); + finish(sqe, file, user_data); + return true; +} + +bool UringRing::recv(RingFile file, void* buffer, size_t length, int flags, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_recv(sqe, target(file), buffer, length, flags); + finish(sqe, file, user_data); + return true; +} + +bool UringRing::read_fixed(RingFile file, void* buffer, size_t length, unsigned buffer_slot, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_read_fixed(sqe, target(file), buffer, static_cast(length), 0, + static_cast(buffer_slot)); + finish(sqe, file, user_data); + return true; +} + +bool UringRing::recvmsg(RingFile file, msghdr* message, int flags, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_recvmsg(sqe, target(file), message, static_cast(flags)); + finish(sqe, file, user_data); + return true; +} + +bool UringRing::send(RingFile file, const void* buffer, size_t length, int flags, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_send(sqe, target(file), buffer, length, flags); + finish(sqe, file, user_data); + return true; +} + +bool UringRing::sendmsg(RingFile file, const msghdr* message, int flags, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_sendmsg(sqe, target(file), message, static_cast(flags)); + finish(sqe, file, user_data); + return true; +} + +bool UringRing::read_eventfd(int fd, uint64_t* value, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_read(sqe, fd, value, sizeof(*value), 0); + io_uring_sqe_set_data64(sqe, user_data); + return true; +} + +bool UringRing::cancel(uint64_t target_data, uint64_t user_data) { + struct io_uring_sqe* sqe = next_sqe(state_->ring); + if (sqe == nullptr) return false; + io_uring_prep_cancel64(sqe, target_data, 0); + io_uring_sqe_set_data64(sqe, user_data); + return true; +} + +int UringRing::submit() { + return io_uring_submit(&state_->ring); +} + +int UringRing::reap(std::vector& out) { + unsigned head; + unsigned seen = 0; + struct io_uring_cqe* cqe; + io_uring_for_each_cqe(&state_->ring, head, cqe) { + ++seen; + if (cqe->user_data == LIBURING_UDATA_TIMEOUT) { + continue; + } + out.push_back(RingCompletion{cqe->user_data, cqe->res, cqe->flags}); + } + if (seen > 0) { + io_uring_cq_advance(&state_->ring, seen); // once for the whole batch + } + return static_cast(seen); +} + +int UringRing::wait(std::vector& out, unsigned wanted, int64_t timeout_us, int64_t spin_us) { + const size_t before = out.size(); + auto have = [&] { return static_cast(out.size() - before); }; + if (spin_us > 0) { + const int rc = io_uring_submit(&state_->ring); + if (rc < 0 && rc != -EAGAIN && rc != -EBUSY && rc != -EINTR) { + return rc; + } + const int64_t start = now_us(); + const int64_t budget = spin_us < timeout_us ? spin_us : timeout_us; + for (;;) { + reap(out); + if (have() >= wanted) { + return static_cast(have()); + } + if (now_us() - start >= budget) { + break; + } + cpu_relax(); + } + timeout_us -= now_us() - start; + if (timeout_us <= 0) { + return static_cast(have()); + } + } + reap(out); + if (have() < wanted) { + struct __kernel_timespec ts{}; + ts.tv_sec = timeout_us / 1'000'000; + ts.tv_nsec = (timeout_us % 1'000'000) * 1000; + struct io_uring_cqe* cqe = nullptr; + const int rc = io_uring_submit_and_wait_timeout(&state_->ring, &cqe, wanted - have(), &ts, nullptr); + if (rc < 0 && rc != -ETIME && rc != -EINTR && rc != -EAGAIN && rc != -EBUSY) { + return rc; + } + reap(out); + } else { + // Enough is there already; still hand over what was prepared. + const int rc = io_uring_submit(&state_->ring); + if (rc < 0 && rc != -EAGAIN && rc != -EBUSY && rc != -EINTR) { + return rc; + } + } + return static_cast(have()); +} + +} // namespace libe3 diff --git a/src/connector/uring_ring.hpp b/src/connector/uring_ring.hpp new file mode 100644 index 00000000..a42b15c3 --- /dev/null +++ b/src/connector/uring_ring.hpp @@ -0,0 +1,224 @@ +/** + * @file uring_ring.hpp + * @brief One io_uring instance, without liburing in the interface + * + * The connector speaks to the ring through this class only, so is included by + * uring_ring.cpp alone. Completions come back as plain RingCompletion values. + * + * Not thread-safe: the caller lets one thread at a time prepare, submit and reap (the + * connector holds a mutex per ring). Registering files and buffers may come from any thread. + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef LIBE3_URING_RING_HPP +#define LIBE3_URING_RING_HPP + +#include +#include +#include +#include +#include +#include + +namespace libe3 { + +/** @brief A completion, as the ring reported it. */ +struct RingCompletion { + uint64_t user_data; ///< What the operation was prepared with + int32_t res; ///< Its result: bytes or a descriptor, or -errno + uint32_t flags; ///< IORING_CQE_F_* flags +}; + +/** @brief How a ring is set up. */ +struct RingOptions { + unsigned depth{64}; ///< Submission queue entries; the completion queue has four times as many + bool sqpoll{false}; ///< A kernel thread takes the submissions, so submitting needs no system call + int sqpoll_cpu{-1}; ///< The CPU of that thread, or -1 to let the scheduler choose + unsigned sqpoll_idle_ms{1000}; ///< How long that thread polls before it sleeps + unsigned file_slots{0}; ///< Registered-file slots to reserve; 0 for none + unsigned buffer_slots{0}; ///< Registered-buffer slots to reserve; 0 for none +}; + +/** @brief A socket an operation acts on: by descriptor, or by registered-file slot. */ +struct RingFile { + int fd; ///< The descriptor + int slot; ///< Its registered-file slot, or -1 to use the descriptor +}; + +/** @brief One io_uring instance. */ +class UringRing { +public: + /** + * @brief Set up a ring. + * @param options How + * @param share_kernel_thread_with A ring whose kernel submission thread this one uses (with sqpoll), or null + * @param error Set to the reason when the ring cannot be set up + * @param error_code Set to the errno of that reason + * @return The ring, or null + */ + static std::unique_ptr create(const RingOptions& options, const UringRing* share_kernel_thread_with, + std::string* error, int* error_code); + + /** @brief Tear the ring down; the kernel cancels what is still in flight. */ + ~UringRing(); + + UringRing(const UringRing&) = delete; + UringRing& operator=(const UringRing&) = delete; + + /** + * @brief Submission queue entries the kernel granted. + * @return The number of entries + */ + unsigned sq_entries() const noexcept; + /** + * @brief Completion queue entries the kernel granted. + * @return The number of entries + */ + unsigned cq_entries() const noexcept; + /** + * @brief Whether a kernel thread takes the submissions. + * @return true with IORING_SETUP_SQPOLL + */ + bool sqpoll() const noexcept { return sqpoll_; } + /** + * @brief Registered-file slots reserved. + * @return Their number, 0 when none were asked for or the kernel refused them + */ + unsigned file_slots() const noexcept { return file_slots_; } + /** + * @brief Registered-buffer slots reserved. + * @return Their number, 0 when none were asked for or the kernel refused them + */ + unsigned buffer_slots() const noexcept { return buffer_slots_; } + + /** + * @brief Put a descriptor in a registered-file slot, or empty the slot. + * @param slot The slot + * @param fd The descriptor, or -1 to empty it + * @return 0 or -errno + */ + int set_file(unsigned slot, int fd); + /** + * @brief Register a buffer in a slot, or empty the slot. + * @param slot The slot + * @param base The buffer, or null to empty it + * @param len Its length + * @return 0 or -errno + */ + int set_buffer(unsigned slot, void* base, size_t len); + + /** + * @brief Prepare an accept. + * @param file The listening socket + * @param address Where the peer's address goes; must stay valid until the completion + * @param length Its room, then its length; must stay valid until the completion + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool accept(RingFile file, sockaddr* address, socklen_t* length, uint64_t user_data); + /** + * @brief Prepare a receive. + * @param file The socket + * @param buffer Where the bytes go; must stay valid until the completion + * @param length Its room + * @param flags recv flags + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool recv(RingFile file, void* buffer, size_t length, int flags, uint64_t user_data); + /** + * @brief Prepare a receive into a registered buffer. + * @param file The socket + * @param buffer Where the bytes go, inside the registered buffer + * @param length Its room + * @param buffer_slot The slot of the registered buffer + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool read_fixed(RingFile file, void* buffer, size_t length, unsigned buffer_slot, uint64_t user_data); + /** + * @brief Prepare a recvmsg. + * @param file The socket + * @param message The message header; the kernel writes its flags and control data back, so it + * must stay valid until the completion + * @param flags recvmsg flags + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool recvmsg(RingFile file, msghdr* message, int flags, uint64_t user_data); + /** + * @brief Prepare a send. + * @param file The socket + * @param buffer The bytes; must stay valid until the completion + * @param length How many + * @param flags send flags + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool send(RingFile file, const void* buffer, size_t length, int flags, uint64_t user_data); + /** + * @brief Prepare a sendmsg. + * @param file The socket + * @param message The message; it and what it points to must stay valid until the completion + * @param flags sendmsg flags + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool sendmsg(RingFile file, const msghdr* message, int flags, uint64_t user_data); + /** + * @brief Prepare a read of an eventfd's eight bytes. + * @param fd The eventfd + * @param value Where the count goes; must stay valid until the completion + * @param user_data What the completion carries + * @return false when the submission queue is full + */ + bool read_eventfd(int fd, uint64_t* value, uint64_t user_data); + /** + * @brief Prepare the cancellation of an operation. + * @param target The user data the operation was prepared with + * @param user_data What the cancellation's own completion carries + * @return false when the submission queue is full + */ + bool cancel(uint64_t target, uint64_t user_data); + + /** + * @brief Hand what was prepared to the kernel. + * @return How many, or -errno + */ + int submit(); + + /** + * @brief Submit what was prepared, then gather completions. + * + * Returns once at least @p wanted completions are there or @p timeout_us has passed. For + * the first @p spin_us of that it polls the completion queue instead of sleeping. + * + * @param out Completions are appended here + * @param wanted How many to wait for + * @param timeout_us How long to wait at most, in microseconds + * @param spin_us How long to poll before sleeping, in microseconds + * @return How many were appended, or -errno for an error of the ring itself + */ + int wait(std::vector& out, unsigned wanted, int64_t timeout_us, int64_t spin_us); + + /** + * @brief Append the completions that are there, without waiting. + * @param out Where they are appended + * @return How many the queue held + */ + int reap(std::vector& out); + +private: + UringRing(); + struct State; + std::unique_ptr state_; + bool sqpoll_{false}; + unsigned file_slots_{0}; + unsigned buffer_slots_{0}; +}; + +} // namespace libe3 + +#endif // LIBE3_URING_RING_HPP diff --git a/tests/test_swig_smoke.py b/tests/test_swig_smoke.py index f9ceb349..005ff5ab 100644 --- a/tests/test_swig_smoke.py +++ b/tests/test_swig_smoke.py @@ -42,6 +42,12 @@ def main() -> int: print("FAIL: libe3py.E3Role_DAPP not exposed", file=sys.stderr) return 1 + # Every link layer is exposed, with the number the C API uses + for name, value in (("E3LinkLayer_ZMQ", 0), ("E3LinkLayer_POSIX", 1), ("E3LinkLayer_URING", 2)): + if getattr(libe3py, name, None) != value: + print(f"FAIL: libe3py.{name} = {getattr(libe3py, name, None)!r}, expected {value}", file=sys.stderr) + return 1 + # E3Config is constructible and has the role field cfg = libe3py.E3Config() if cfg.role != libe3py.E3Role_RAN: diff --git a/tests/test_types.cpp b/tests/test_types.cpp index e9aa6d1f..702f5fc0 100644 --- a/tests/test_types.cpp +++ b/tests/test_types.cpp @@ -193,6 +193,8 @@ TEST(E3Config_zmq) { TEST(E3LinkLayer_to_string) { ASSERT_STREQ(link_layer_to_string(E3LinkLayer::ZMQ), "zmq"); ASSERT_STREQ(link_layer_to_string(E3LinkLayer::POSIX), "posix"); + ASSERT_STREQ(link_layer_to_string(E3LinkLayer::URING), "uring"); + ASSERT_EQ(static_cast(E3LinkLayer::URING), 2); // the C API and the Python binding pass the number } TEST(E3TransportLayer_to_string) { From 30322da6fd2d6e4f95b3ee98f54375f659c37b2b Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Wed, 11 Nov 2026 15:25:59 -0500 Subject: [PATCH 4/7] tests: run the one-endpoint tests over both socket links, test the io_uring link, compare the links The tests of the one-endpoint wire take --ran-link and --dapp-link (tests/link_under_test.hpp) and run once per pair in CTest: both POSIX, both io_uring and each mixed pair. Where the kernel refuses io_uring they exit 77, which CTest reports as skipped, unless LIBE3_REQUIRE_URING is set, so that a machine meant to run io_uring cannot pass by skipping. test_uring_connector covers the ring options, a broadcast to more peers than the ring has entries, a stalled peer, a reset peer, 2000 PDUs of every size on every transport, the PDU limit, the receive timeout and the wake of a waiting receive by shutdown. The ports of these tests now come from free_ports(): below the range of source ports, free when asked, and claimed with a lock on a file in /tmp/libe3-test-ports that the process holds until it exits, so that two tests never take the same port however they order asking and listening. The tests chose their ports from the range the kernel also uses for the source ports of outgoing connections, and each from a formula on its pid with a base and a width of its own, so the bands overlapped. Once enough connections had come and gone the listen of a later test met a source port still in TIME_WAIT (2 runs in 30 of test_uring_connector), and two tests with overlapping bands met each other (8 runs in 12 of the connector subset at -j3, always test_role_pair_posix or a variant). 0 in 80 and 0 in 12 after. The link-matrix entries run with LATREC_ENTRIES_LOG2=12: every agent thread leaves an 8 MiB pre-faulted latrec ring in /latrec, and repeating the matrix filled a disk. test_bench_full_loop_latency takes --link uring, --iterations, --warmup and CPU pinning, and reports the CPU and context switches per round trip. tools/bench_matrix.py runs it over links, transports and encodings in interleaved rounds and names the fastest link per transport only when its slowest round beats every other link's best; its parsing and verdict are tested on canned output (test_bench_matrix.py). The benchmark pairs the dApp's inbound records with the RAN's outbound ones by position and checked that each pair was in order; a receiver that polls can have a PDU before the sender has stamped its send as done (SCTP over loopback delivers inside the send call), so about one run in ten of SCTP with busy polling was refused as misaligned although the counts of PDUs out and in agreed. The check now allows 5 us of skew (a pair that is off by one round trip is off by 12 us or more), the phase arithmetic is signed, and a refusal says which pair was first and how many groups there were. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- cmake/libe3Tests.cmake | 70 ++- tests/integration/bench_full_loop_latency.cpp | 115 +++- tests/integration/test_latrec_stages.cpp | 12 +- .../integration/test_multi_peer_dispatch.cpp | 53 +- tests/integration/test_role_pair_posix.cpp | 15 +- tests/integration/test_role_pair_sctp.cpp | 23 +- tests/link_under_test.hpp | 227 ++++++++ tests/test_bench_matrix.py | 96 ++++ tests/test_dapp_bound_to_connection_posix.cpp | 11 +- tests/test_dapp_link_closed_posix.cpp | 7 +- tests/test_dapp_ran_loss.cpp | 6 + tests/test_dapp_reconnect.cpp | 6 + tests/test_dapp_refused_setup_reconnect.cpp | 17 +- tests/test_link_availability.cpp | 120 ++++ tests/test_role_aware_connector_posix.cpp | 16 +- tests/test_uring_connector.cpp | 540 ++++++++++++++++++ tools/bench_matrix.py | 230 ++++++++ 17 files changed, 1494 insertions(+), 70 deletions(-) create mode 100644 tests/link_under_test.hpp create mode 100644 tests/test_bench_matrix.py create mode 100644 tests/test_link_availability.cpp create mode 100644 tests/test_uring_connector.cpp create mode 100644 tools/bench_matrix.py diff --git a/cmake/libe3Tests.cmake b/cmake/libe3Tests.cmake index bc27283e..740a5f2f 100644 --- a/cmake/libe3Tests.cmake +++ b/cmake/libe3Tests.cmake @@ -44,6 +44,49 @@ set(LIBE3_ZMQ_ONLY_TESTS setup_bad_request ) +# Tests that drive the io_uring connector itself (it includes src/connector/uring_connector.hpp): +# built only where the io_uring link is (LIBE3_WITH_URING, see libe3Dependencies.cmake). +set(LIBE3_URING_ONLY_TESTS + uring_connector +) + +# The POSIX and the io_uring link speak the same wire, so these tests of the one-endpoint wire +# run once per pair of links: both POSIX (the plain test), both io_uring, and each mixed pair. +# The binary takes --ran-link/--dapp-link (tests/link_under_test.hpp) and exits 77, which CTest +# reports as skipped, where the kernel refuses io_uring; LIBE3_REQUIRE_URING turns that into a +# failure. Unit tests here; the integration ones are in LIBE3_LINK_MATRIX_INTEGRATION_TESTS. +set(LIBE3_LINK_MATRIX_TESTS + dapp_bound_to_connection_posix + dapp_link_closed_posix + role_aware_connector_posix +) +set(LIBE3_LINK_MATRIX_INTEGRATION_TESTS + role_pair_posix + role_pair_sctp +) + +# Adds _uring, _uring_ran_posix_dapp and _posix_ran_uring_dapp. +function(libe3_add_link_matrix target_name) + if(NOT LIBE3_WITH_URING) + return() + endif() + get_test_property(${target_name} LABELS _labels) + foreach(_pair "uring;uring;_uring" "uring;posix;_uring_ran_posix_dapp" "posix;uring;_posix_ran_uring_dapp") + list(GET _pair 0 _ran) + list(GET _pair 1 _dapp) + list(GET _pair 2 _suffix) + add_test(NAME ${target_name}${_suffix} + COMMAND ${target_name} --ran-link ${_ran} --dapp-link ${_dapp}) + # A latrec build leaves a ring per agent thread in the build tree (pre-faulted, 8 MiB by + # default) and these runs read none of them: small rings keep the matrix from piling up disk. + set_tests_properties(${target_name}${_suffix} PROPERTIES SKIP_RETURN_CODE 77 + ENVIRONMENT "LATREC_ENTRIES_LOG2=12") + if(_labels) + set_tests_properties(${target_name}${_suffix} PROPERTIES LABELS "${_labels}") + endif() + endforeach() +endfunction() + # E2SM-DAPP codec tests (tests/test_e2sm_dapp_*.cpp). They link libe3::e2sm_dapp # alone, which is the point: the codec must work without libe3, so only the # link-coexist test also links libe3. Each of encode/decode/roundtrip/c_api is @@ -86,6 +129,10 @@ foreach(test_src IN LISTS LIBE3_TEST_SOURCES) message(STATUS "Skipping test_${simple_name}: ZeroMQ support disabled") continue() endif() + if(NOT LIBE3_WITH_URING AND simple_name IN_LIST LIBE3_URING_ONLY_TESTS) + message(STATUS "Skipping test_${simple_name}: io_uring link not built") + continue() + endif() if(simple_name MATCHES "^e2sm_dapp_") if(NOT LIBE3_ENABLE_E2SM_DAPP) message(STATUS "Skipping test_${simple_name}: E2SM-DAPP codec not built") @@ -137,12 +184,18 @@ foreach(test_src IN LISTS LIBE3_TEST_SOURCES) LIBE3_FIXTURE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures") endif() - if(simple_name STREQUAL "socket_link") - # It reaches into the connectors' own header: the framing. + if(simple_name IN_LIST LIBE3_URING_ONLY_TESTS OR simple_name STREQUAL "socket_link") + # They reach into the connectors' own headers: the ring settings in effect, the framing. target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) endif() add_test(NAME ${target_name} COMMAND ${target_name}) + if(simple_name IN_LIST LIBE3_URING_ONLY_TESTS) + set_tests_properties(${target_name} PROPERTIES SKIP_RETURN_CODE 77) + endif() + if(simple_name IN_LIST LIBE3_LINK_MATRIX_TESTS) + libe3_add_link_matrix(${target_name}) + endif() endforeach() # Integration tests in tests/integration/ are opt-in via @@ -205,6 +258,9 @@ if(LIBE3_BUILD_INTEGRATION_TESTS AND LIBE3_ENABLE_ASN1) if(simple_name MATCHES "sctp") set_tests_properties(${target_name} PROPERTIES SKIP_RETURN_CODE 77) endif() + if(simple_name IN_LIST LIBE3_LINK_MATRIX_INTEGRATION_TESTS) + libe3_add_link_matrix(${target_name}) + endif() endforeach() endif() @@ -226,6 +282,16 @@ else() message(STATUS "Skipping test_latrec_reader: no Python 3 interpreter") endif() +# --- the link comparison of the full-loop benchmark --------------------------- +# tools/bench_matrix.py names the fastest link in the CI report; its parsing and its verdict are +# checked on canned benchmark output, so they do not depend on a benchmark run. +if(Python3_Interpreter_FOUND) + add_test(NAME test_bench_matrix + COMMAND ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_bench_matrix.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/bench_matrix.py") +endif() + # --- latrec2csv against injected ground truth --------------------------------- # Synthetic rings whose hops are known in advance, so the CSVs are checked # against the delays that were written rather than against themselves. Needs diff --git a/tests/integration/bench_full_loop_latency.cpp b/tests/integration/bench_full_loop_latency.cpp index 231e5c15..2031325e 100644 --- a/tests/integration/bench_full_loop_latency.cpp +++ b/tests/integration/bench_full_loop_latency.cpp @@ -72,6 +72,14 @@ * chronological position instead, the same technique this file has always * used for DELIVER_BEGIN. * + * Options: --link zmq|posix|uring, --transport ipc|tcp|sctp, --encoding asn1|json|protobuf, + * --iterations N (default 1000) and --warmup N (default 50), and --ran-cpu/--dapp-cpu to pin + * each agent's I/O threads (E3Config::io_thread_affinity). The io_uring link reads its ring + * options from LIBE3_URING_* (docs/connectors/uring.md); the ones set are printed under the + * table, with the CPU time the process spent per round trip, both ends together, because a + * link that polls instead of sleeping buys latency with CPU. tools/bench_matrix.py runs this + * over links, transports and encodings and compares them. + * * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University * SPDX-License-Identifier: Apache-2.0 */ @@ -93,6 +101,7 @@ #include #include #include +#include #include #include #include @@ -104,8 +113,8 @@ using clk = steady_clock; namespace { -constexpr int kIterations = 1000; -constexpr int kWarmupIterations = 50; +int kIterations = 1000; +int kWarmupIterations = 50; constexpr uint32_t kRanFunctionId = 1; constexpr uint32_t kControlId = 1; @@ -147,6 +156,7 @@ std::string make_trace_dir() { E3LinkLayer parse_link(const char* s) { if (std::strcmp(s, "posix") == 0) return E3LinkLayer::POSIX; if (std::strcmp(s, "zmq") == 0) return E3LinkLayer::ZMQ; + if (std::strcmp(s, "uring") == 0) return E3LinkLayer::URING; std::fprintf(stderr, "Unknown link layer '%s'; using zmq\n", s); return E3LinkLayer::ZMQ; } @@ -310,8 +320,15 @@ std::vector reconstruct(const std::string& dir) { return a.decode_e3ap_done < b.decode_e3ap_done; }); + // The sender stamps SEND_DONE when its send call has returned, and a receiver that polls can have + // the PDU and stamp RECV before that: over loopback, SCTP delivers inside the sender's system call, + // and io_uring completes a send on a kernel thread. Such a hop is a little below zero. The + // pairing check below tolerates it by this much; a pairing that is off by one round trip is off by + // the whole round trip, 12 us or more even for the fastest link, so it is still caught. + constexpr uint64_t kStampSkewNs = 5000; std::vector out; size_t skipped_misaligned = 0; + char first_misaligned[400] = ""; size_t ind_inbound_idx = 0, ctrl_inbound_idx = 0; // std::map iterates in ascending key order, i.e. ascending business seq, // i.e. emission order -- the same order PingPong pacing guarantees the @@ -356,9 +373,24 @@ std::vector reconstruct(const std::string& dir) { // one -- checking that the two joins agree on basic time ordering // catches that and drops just the affected round trip, the same // tolerance this file already extends to an incomplete tail. - const bool ind_ok = io.send_done <= ii.recv && ii.deliver_begin <= *t5; - const bool ctrl_ok = co.send_done <= ci.recv && ci.decode_e3ap_done <= *t8; + const bool ind_ok = io.send_done <= ii.recv + kStampSkewNs && ii.deliver_begin <= *t5; + const bool ctrl_ok = co.send_done <= ci.recv + kStampSkewNs && ci.decode_e3ap_done <= *t8; if (!ind_ok || !ctrl_ok) { + if (skipped_misaligned == 0) { + // The first one tells a lost or extra inbound group (every later round trip is off by + // one) from one whose two stamps merely landed in the other order. Microseconds from + // the indication's emission. + const auto rel = [&](uint64_t t) { + return static_cast(static_cast(t) - static_cast(io.emit_enter)) / 1000.0; + }; + std::snprintf(first_misaligned, sizeof(first_misaligned), + "first at business seq %llu (indication #%zu, control #%zu): ind send_done %.2f recv %.2f " + "deliver_begin %.2f decode_e3sm_done %.2f | ctrl send_done %.2f recv %.2f " + "decode_e3ap_done %.2f decode_e3sm_begin %.2f", + static_cast(seq), ind_inbound_idx, ctrl_inbound_idx, + rel(io.send_done), rel(ii.recv), rel(ii.deliver_begin), rel(*t5), + rel(co.send_done), rel(ci.recv), rel(ci.decode_e3ap_done), rel(*t8)); + } ++skipped_misaligned; continue; } @@ -396,8 +428,10 @@ std::vector reconstruct(const std::string& dir) { if (skipped_misaligned > 0) { std::fprintf(stderr, "WARNING: dropped %zu round trip(s) with a misaligned " - "inbound position (see reconstruct()'s comment)\n", - skipped_misaligned); + "inbound position (see reconstruct()'s comment)\n" + " groups: %zu indications and %zu controls out, %zu indications and %zu controls in; %s\n", + skipped_misaligned, ind_outbound.size(), ctrl_outbound.size(), ind_inbound.size(), + ctrl_inbound.size(), first_misaligned); } return out; } @@ -409,23 +443,34 @@ int main(int argc, char* argv[]) { E3LinkLayer link = E3LinkLayer::ZMQ; E3TransportLayer transport = E3TransportLayer::IPC; EncodingFormat encoding = EncodingFormat::ASN1; + int ran_cpu = -1; + int dapp_cpu = -1; static const struct option long_opts[] = { - {"link", required_argument, nullptr, 'l'}, - {"transport", required_argument, nullptr, 't'}, - {"encoding", required_argument, nullptr, 'e'}, - {"help", no_argument, nullptr, 'h'}, - {nullptr, 0, nullptr, 0}, + {"link", required_argument, nullptr, 'l'}, + {"transport", required_argument, nullptr, 't'}, + {"encoding", required_argument, nullptr, 'e'}, + {"iterations", required_argument, nullptr, 'n'}, + {"warmup", required_argument, nullptr, 'w'}, + {"ran-cpu", required_argument, nullptr, 'r'}, + {"dapp-cpu", required_argument, nullptr, 'd'}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0}, }; int opt; - while ((opt = getopt_long(argc, argv, "l:t:e:h", long_opts, nullptr)) != -1) { + while ((opt = getopt_long(argc, argv, "l:t:e:n:w:r:d:h", long_opts, nullptr)) != -1) { switch (opt) { case 'l': link = parse_link(optarg); break; case 't': transport = parse_transport(optarg); break; case 'e': encoding = parse_encoding(optarg); break; + case 'n': kIterations = std::max(1, std::atoi(optarg)); break; + case 'w': kWarmupIterations = std::max(0, std::atoi(optarg)); break; + case 'r': ran_cpu = std::atoi(optarg); break; + case 'd': dapp_cpu = std::atoi(optarg); break; case 'h': - std::printf("Usage: %s [--link zmq|posix] [--transport ipc|tcp|sctp]" - " [--encoding asn1|json|protobuf]\n", argv[0]); + std::printf("Usage: %s [--link zmq|posix|uring] [--transport ipc|tcp|sctp]" + " [--encoding asn1|json|protobuf] [--iterations N] [--warmup N]" + " [--ran-cpu CPU] [--dapp-cpu CPU]\n", argv[0]); return 0; default: std::fprintf(stderr, "Unknown option; use --help\n"); @@ -441,7 +486,10 @@ int main(int argc, char* argv[]) { // count, against the default 4M-record capacity. const std::string trace_dir = make_trace_dir(); latrec_set_output_dir(trace_dir.c_str()); - setenv("LATREC_ENTRIES_LOG2", "16", 1); + // About 30 records per round trip land in the busiest ring; keep the ring from wrapping. + int entries_log2 = 16; + while (entries_log2 < 22 && (1L << entries_log2) < 40L * (kIterations + kWarmupIterations)) ++entries_log2; + setenv("LATREC_ENTRIES_LOG2", std::to_string(entries_log2).c_str(), 1); E3Config ran_cfg; ran_cfg.role = E3Role::RAN; @@ -450,6 +498,7 @@ int main(int argc, char* argv[]) { ran_cfg.transport_layer = transport; ran_cfg.encoding = encoding; ran_cfg.log_level = 0; + ran_cfg.io_thread_affinity = ran_cpu; // IPC transport: use a private tmpdir so the benchmark is self-contained. // TCP/SCTP: both sides run in the same process on localhost; the default @@ -465,6 +514,7 @@ int main(int argc, char* argv[]) { auto dapp_cfg = ran_cfg; dapp_cfg.role = E3Role::DAPP; dapp_cfg.dapp_name = "BenchDApp"; + dapp_cfg.io_thread_affinity = dapp_cpu; // For IPC the dApp inherits the same explicit endpoints from ran_cfg. // For TCP/SCTP the dApp connects to localhost on the default ports, // which is correct since both sides run in the same process. @@ -538,7 +588,10 @@ int main(int argc, char* argv[]) { // an incomplete tail; reconstruct() below drops it rather than treat it // as a real sample, which costs at most one sample out of the total. const int total = kIterations + kWarmupIterations; - auto deadline = clk::now() + std::chrono::seconds(60); + struct rusage usage_before{}; + getrusage(RUSAGE_SELF, &usage_before); + const int handled_before = handled.load(); + auto deadline = clk::now() + std::chrono::seconds(60 + total / 500); while (true) { if (handled.load(std::memory_order_relaxed) >= total) break; if (clk::now() > deadline) { @@ -549,6 +602,10 @@ int main(int argc, char* argv[]) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); } + struct rusage usage_after{}; + getrusage(RUSAGE_SELF, &usage_after); + const int handled_in_window = handled.load() - handled_before; + dapp.stop(); ran.stop(); latrec_set_output_dir(nullptr); @@ -574,8 +631,9 @@ int main(int argc, char* argv[]) { p14, p15, p16, p17, p18, total_us; for (const auto& t : round_trips) { if (t.seq < static_cast(kWarmupIterations)) continue; + // Signed: a hop can come out below zero (see kStampSkewNs). auto us = [](uint64_t a, uint64_t b) { - return static_cast(b - a) / 1000.0; + return static_cast(static_cast(b) - static_cast(a)) / 1000.0; }; p1.push_back(us(t.t1, t.t2)); p2.push_back(us(t.t2, t.t3)); @@ -664,5 +722,28 @@ int main(int argc, char* argv[]) { "`DECODE_E3SM_BEGIN` to `DECODE_E3SM_DONE`", p18); emit_row("Total", "**Total round-trip**", "", total_us); std::printf("\n"); + + auto tv_us = [](const timeval& tv) { + return static_cast(tv.tv_sec) * 1e6 + static_cast(tv.tv_usec); + }; + const double rounds = std::max(1, handled_in_window); + const double user_us = (tv_us(usage_after.ru_utime) - tv_us(usage_before.ru_utime)) / rounds; + const double sys_us = (tv_us(usage_after.ru_stime) - tv_us(usage_before.ru_stime)) / rounds; + const double voluntary = static_cast(usage_after.ru_nvcsw - usage_before.ru_nvcsw) / rounds; + const double involuntary = static_cast(usage_after.ru_nivcsw - usage_before.ru_nivcsw) / rounds; + std::printf("CPU per round trip, both ends: %.2f us user, %.2f us system; " + "context switches per round trip: %.2f voluntary, %.2f involuntary.\n", + user_us, sys_us, voluntary, involuntary); + std::string ring_options; + for (const char* name : {"LIBE3_URING_DEPTH", "LIBE3_URING_FIXED_FILES", "LIBE3_URING_FIXED_BUFFERS", + "LIBE3_URING_SQPOLL", "LIBE3_URING_SQPOLL_CPU", "LIBE3_URING_BUSY_POLL_US"}) { + if (const char* value = std::getenv(name)) { + ring_options += std::string(ring_options.empty() ? "" : ", ") + name + "=" + value; + } + } + if (link == E3LinkLayer::URING) { + std::printf("io_uring ring options: %s.\n", ring_options.empty() ? "the defaults" : ring_options.c_str()); + } + std::printf("\n"); return p1.empty() ? 1 : 0; } diff --git a/tests/integration/test_latrec_stages.cpp b/tests/integration/test_latrec_stages.cpp index 9b6422c1..533a1293 100644 --- a/tests/integration/test_latrec_stages.cpp +++ b/tests/integration/test_latrec_stages.cpp @@ -18,6 +18,7 @@ #include "sm_simple/simple_service_model.hpp" #include "latrec_ring_reader.hpp" #include "test_framework.hpp" +#include "link_under_test.hpp" #include #include @@ -438,12 +439,21 @@ int main() { // The POSIX RAN send() accepts new peers and broadcasts to all of them, // which the ZMQ one does not; JSON and Protobuf change what L1->L2 and // L4->L5 measure. - const Combo combos[] = { + // The io_uring link stamps SEND_DONE when the kernel has the bytes, as POSIX does. + std::vector combos = { {"zmq-ipc-asn1", E3LinkLayer::ZMQ, E3TransportLayer::IPC, EncodingFormat::ASN1}, {"posix-ipc-asn1", E3LinkLayer::POSIX, E3TransportLayer::IPC, EncodingFormat::ASN1}, {"zmq-ipc-json", E3LinkLayer::ZMQ, E3TransportLayer::IPC, EncodingFormat::JSON}, {"zmq-ipc-protobuf", E3LinkLayer::ZMQ, E3TransportLayer::IPC, EncodingFormat::PROTOBUF}, }; + if (libe3_test::uring_available()) { + combos.push_back({"uring-ipc-asn1", E3LinkLayer::URING, E3TransportLayer::IPC, EncodingFormat::ASN1}); + } else if (libe3_test::uring_required()) { + std::fprintf(stderr, "FAIL: io_uring is required (LIBE3_REQUIRE_URING) and cannot be used here\n"); + return 1; + } else { + std::printf("URING NOT RUN: this build or this kernel cannot create an io_uring ring\n"); + } for (const Combo& c : combos) { const size_t before = read_ring_dir(trace_dir).size(); diff --git a/tests/integration/test_multi_peer_dispatch.cpp b/tests/integration/test_multi_peer_dispatch.cpp index 914f370c..cc7c69ac 100644 --- a/tests/integration/test_multi_peer_dispatch.cpp +++ b/tests/integration/test_multi_peer_dispatch.cpp @@ -14,15 +14,18 @@ * is the same property that matters when each E3Agent connects to a * different remote RAN. * - * The scenario runs over BOTH link layers: ZMQ (PUB/SUB broadcast) and - * POSIX (multi-peer accept loops + broadcast fan-out), which must deliver - * the same no-crosstalk semantics. + * The scenario runs over every link layer: ZMQ (PUB/SUB broadcast), POSIX + * (multi-peer accept loops + broadcast fan-out) and io_uring (the same wire as + * POSIX, one submission for the fan-out), which must deliver the same + * no-crosstalk semantics. The io_uring cases say URING NOT RUN where this + * build or kernel has no io_uring (link_under_test.hpp). * * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University * SPDX-License-Identifier: Apache-2.0 */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include #include #include "sm_simple/e3sm_simple_wrapper.hpp" @@ -58,17 +61,16 @@ std::string make_tmpdir() { return std::string(d); } -// A literal base port collides across concurrent CTest runs on a shared, -// multi-session machine (this repo's normal state). LIBE3_TEST_PORT_OFFSET +// A fixed base port collides across concurrent CTest runs on a shared, +// multi-session machine (this repo's normal state), and a base derived from the +// pid collides with the other tests' own derivations. LIBE3_TEST_PORT_OFFSET // lets a CI runner assign each parallel job a disjoint range explicitly; -// absent that, spread by pid so co-resident local runs don't collide either. -// *3 keeps one process's whole triple (base, base+1, base+2) from -// overlapping another process's triple. +// absent that, the triple (base, base+1, base+2) is reserved by free_ports(). uint16_t unique_base_port(uint16_t default_base) { if (const char* env = std::getenv("LIBE3_TEST_PORT_OFFSET")) { return static_cast(default_base + std::atoi(env)); } - return static_cast(default_base + (static_cast(getpid()) % 1000) * 3); + return libe3_test::free_ports(3); } // Connects to 127.0.0.1:port, retrying with a short bounded backoff. Needed @@ -448,26 +450,31 @@ TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_posix_ipc) { } TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_posix_tcp) { - // Unique port triple: 23990-23992 and 24990-24992 are used by other POSIX - // tests; 25990-25992 is free. unique_base_port() also keeps it distinct - // across concurrent runs on a shared machine. + // unique_base_port() reserves a port triple of its own. run_multi_peer_two_dapps(E3LinkLayer::POSIX, E3TransportLayer::TCP, /*base_port=*/unique_base_port(25990)); } +TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_uring_ipc) { + if (!libe3_test::uring_runs_here()) return; + run_multi_peer_two_dapps(E3LinkLayer::URING, E3TransportLayer::IPC); +} + +TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_uring_tcp) { + if (!libe3_test::uring_runs_here()) return; + run_multi_peer_two_dapps(E3LinkLayer::URING, E3TransportLayer::TCP, + /*base_port=*/unique_base_port(27990)); +} + // Regression test for the multi-peer blocker: a dApp that registers, subscribes and then // stops reading must NOT stall delivery to the other peers, and must NOT hang stop(). // Before the SO_SNDTIMEO fix, the RAN's outbound thread blocked forever in send() once the // stalled peer's TCP buffer filled, starving every other peer and wedging stop(). -TEST(multi_peer_slow_peer_does_not_stall_others_posix_tcp) { - // distinct from the port triples above; unique_base_port() also keeps it - // distinct across concurrent runs on a shared machine. - const uint16_t base = unique_base_port(26990); - +void run_slow_peer_does_not_stall_others(E3LinkLayer link, uint16_t base) { E3Config ran_cfg; ran_cfg.role = E3Role::RAN; ran_cfg.ran_identifier = "test-ran-slow"; - ran_cfg.link_layer = E3LinkLayer::POSIX; + ran_cfg.link_layer = link; ran_cfg.transport_layer = E3TransportLayer::TCP; ran_cfg.encoding = EncodingFormat::ASN1; ran_cfg.log_level = 0; @@ -517,6 +524,16 @@ TEST(multi_peer_slow_peer_does_not_stall_others_posix_tcp) { close(stalled); } +TEST(multi_peer_slow_peer_does_not_stall_others_posix_tcp) { + // A port triple of its own, like each test above. + run_slow_peer_does_not_stall_others(E3LinkLayer::POSIX, unique_base_port(26990)); +} + +TEST(multi_peer_slow_peer_does_not_stall_others_uring_tcp) { + if (!libe3_test::uring_runs_here()) return; + run_slow_peer_does_not_stall_others(E3LinkLayer::URING, unique_base_port(28990)); +} + int main() { return RUN_ALL_TESTS(); } diff --git a/tests/integration/test_role_pair_posix.cpp b/tests/integration/test_role_pair_posix.cpp index 9fd2fed6..966de638 100644 --- a/tests/integration/test_role_pair_posix.cpp +++ b/tests/integration/test_role_pair_posix.cpp @@ -3,14 +3,16 @@ * @brief End-to-end RAN + dApp pair over POSIX sockets (IPC + TCP). * * Same scenario as test_role_pair_zmq_ipc.cpp but using the POSIX - * connector. SCTP is not exercised (CI runners lack the sctp kernel - * module). + * connector. SCTP is in test_role_pair_sctp.cpp. CTest runs it again over the + * io_uring link, and with a RAN of one link and a dApp of the other + * (link_under_test.hpp), since both links speak the same wire. * * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University * SPDX-License-Identifier: Apache-2.0 */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include #include "sm_simple/e3sm_simple_wrapper.hpp" @@ -102,7 +104,7 @@ void run_pair(E3TransportLayer transport, uint16_t base_port, E3Config c; c.role = E3Role::RAN; c.ran_identifier = "posix-test-ran"; - c.link_layer = E3LinkLayer::POSIX; + c.link_layer = libe3_test::ran_link(); c.transport_layer = transport; c.encoding = EncodingFormat::ASN1; c.log_level = 0; @@ -119,6 +121,7 @@ void run_pair(E3TransportLayer transport, uint16_t base_port, auto ran_cfg = make_ran_cfg(); auto dapp_cfg = ran_cfg; dapp_cfg.role = E3Role::DAPP; + dapp_cfg.link_layer = libe3_test::dapp_link(); dapp_cfg.dapp_name = "PosixTestDApp"; E3Agent ran(ran_cfg); @@ -210,9 +213,11 @@ TEST(role_pair_posix_ipc_end_to_end) { } TEST(role_pair_posix_tcp_end_to_end) { - run_pair(E3TransportLayer::TCP, /*base_port=*/24990, ""); + // Per process: CTest runs this binary once per pair of links, at the same time. + run_pair(E3TransportLayer::TCP, libe3_test::free_ports(3), ""); } -int main() { +int main(int argc, char** argv) { + if (const int rc = libe3_test::select_links(argc, argv); rc >= 0) return rc; return RUN_ALL_TESTS(); } diff --git a/tests/integration/test_role_pair_sctp.cpp b/tests/integration/test_role_pair_sctp.cpp index 1f156437..aab442ee 100644 --- a/tests/integration/test_role_pair_sctp.cpp +++ b/tests/integration/test_role_pair_sctp.cpp @@ -15,6 +15,9 @@ * - the RAN accepts payload protocol id 0 on stream 0, closes an association whose messages * carry another id, requests two streams, and sets the socket buffers it was told to. * + * CTest runs it again over the io_uring link, and with a RAN of one link and a dApp of the + * other (link_under_test.hpp): both links put the same id and streams on the association. + * * Needs the sctp kernel module. Without it the test says so and exits with the code CTest * reads as "skipped" (77), unless LIBE3_REQUIRE_SCTP is set: then it fails, so that a machine * that is meant to run SCTP cannot pass by skipping. @@ -24,6 +27,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include #include #include @@ -65,10 +69,7 @@ namespace { constexpr uint32_t PPID = 0x00E3A001; -uint16_t unique_port() { - static std::atomic next{0}; - return static_cast(31000 + (getpid() % 2000) * 8 + (next.fetch_add(1) % 8)); -} +uint16_t unique_port() { return libe3_test::free_port(); } bool wait_until(const std::function& pred, std::chrono::milliseconds timeout) { const auto deadline = std::chrono::steady_clock::now() + timeout; @@ -326,7 +327,7 @@ E3Config ran_config(uint16_t port) { E3Config c; c.role = E3Role::RAN; c.ran_identifier = "sctp-test-ran"; - c.link_layer = E3LinkLayer::POSIX; + c.link_layer = libe3_test::ran_link(); c.transport_layer = E3TransportLayer::SCTP; c.encoding = EncodingFormat::ASN1; c.log_level = 0; @@ -345,6 +346,7 @@ struct DAppLog { std::unique_ptr make_dapp(E3Config config, const char* name, DAppLog& log) { config.role = E3Role::DAPP; + config.link_layer = libe3_test::dapp_link(); config.dapp_name = name; auto dapp = std::make_unique(config); dapp->set_indication_handler([&log](const IndicationMessage&) { @@ -515,7 +517,7 @@ TEST(sctp_pair_two_dapps_each_association_carries_only_its_own_indications) { TEST(sctp_ran_accepts_ppid_zero_on_stream_zero_and_requests_two_streams) { const uint16_t port = unique_port(); - auto ran = create_connector(E3LinkLayer::POSIX, E3TransportLayer::SCTP, "", "", "", port, 0, 0, 1, E3Role::RAN); + auto ran = create_connector(libe3_test::ran_link(), E3TransportLayer::SCTP, "", "", "", port, 0, 0, 1, E3Role::RAN); ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); // A peer that asks for ten streams gets two: the RAN's listener allows two. @@ -549,7 +551,7 @@ TEST(sctp_ran_accepts_ppid_zero_on_stream_zero_and_requests_two_streams) { TEST(sctp_ran_closes_an_association_whose_messages_carry_another_ppid) { const uint16_t port = unique_port(); - auto ran = create_connector(E3LinkLayer::POSIX, E3TransportLayer::SCTP, "", "", "", port, 0, 0, 1, E3Role::RAN); + auto ran = create_connector(libe3_test::ran_link(), E3TransportLayer::SCTP, "", "", "", port, 0, 0, 1, E3Role::RAN); ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); int peer = connect_to(port, 2); @@ -577,7 +579,7 @@ TEST(sctp_ran_closes_an_association_whose_messages_carry_another_ppid) { TEST(sctp_ran_takes_a_pdu_of_sixty_thousand_bytes_whole) { const uint16_t port = unique_port(); - auto ran = create_connector(E3LinkLayer::POSIX, E3TransportLayer::SCTP, "", "", "", port, 0, 0, 1, E3Role::RAN); + auto ran = create_connector(libe3_test::ran_link(), E3TransportLayer::SCTP, "", "", "", port, 0, 0, 1, E3Role::RAN); ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); int peer = connect_to(port, 2); ASSERT_TRUE(peer >= 0); @@ -597,7 +599,8 @@ TEST(sctp_ran_takes_a_pdu_of_sixty_thousand_bytes_whole) { ran->dispose(); } -int main() { +int main(int argc, char** argv) { + if (const int rc = libe3_test::select_links(argc, argv); rc >= 0) return rc; // The kernel module may be missing even when the headers are there. int probe = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); if (probe < 0) { @@ -615,7 +618,7 @@ int main() { #else // LIBE3_ENABLE_SCTP -int main() { +int main(int, char**) { const char* require = std::getenv("LIBE3_REQUIRE_SCTP"); if (require != nullptr && *require != '\0' && *require != '0') { std::cerr << "FAIL: SCTP is required (LIBE3_REQUIRE_SCTP) and libe3 was built without it\n"; diff --git a/tests/link_under_test.hpp b/tests/link_under_test.hpp new file mode 100644 index 00000000..b3bdf1c2 --- /dev/null +++ b/tests/link_under_test.hpp @@ -0,0 +1,227 @@ +/** + * @file link_under_test.hpp + * @brief Which socket link a test drives, so one test body covers the POSIX and the io_uring link. + * + * The POSIX and the io_uring link speak the same wire, so the tests of the one-endpoint wire + * run against both, and against a RAN of one with a dApp of the other. A test binary takes + * + * --link posix|uring both ends + * --ran-link posix|uring the RAN end + * --dapp-link posix|uring the dApp end + * + * and asks link_for(role) which connector to build. Without arguments both ends are POSIX. + * + * io_uring can be missing where the build has it: a container under Docker's default seccomp + * profile, a kernel with kernel.io_uring_disabled set, or a kernel older than 5.11. A binary + * asked to run uring there says so and exits 77, which CTest reports as skipped, unless + * LIBE3_REQUIRE_URING is set: then it fails, so that a machine meant to run io_uring cannot + * pass by skipping. A single test that needs io_uring calls uring_runs_here() instead. + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef LIBE3_TEST_LINK_UNDER_TEST_HPP +#define LIBE3_TEST_LINK_UNDER_TEST_HPP + +#include "libe3/e3_connector.hpp" +#include "libe3/types.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace libe3_test { + +struct LinksUnderTest { + libe3::E3LinkLayer ran{libe3::E3LinkLayer::POSIX}; + libe3::E3LinkLayer dapp{libe3::E3LinkLayer::POSIX}; +}; + +inline LinksUnderTest& links_under_test() { + static LinksUnderTest links; + return links; +} + +inline libe3::E3LinkLayer ran_link() { return links_under_test().ran; } +inline libe3::E3LinkLayer dapp_link() { return links_under_test().dapp; } + +inline libe3::E3LinkLayer link_for(libe3::E3Role role) { + return role == libe3::E3Role::RAN ? ran_link() : dapp_link(); +} + +/** + * @brief Whether a TCP port, and an SCTP port where the kernel has SCTP, can be bound right now. + * + * This is the question listen() asks. A bind is refused by any socket that holds the port without + * SO_REUSEADDR, and that includes the client sockets of earlier connections still in TIME_WAIT: + * the kernel hands out source ports from 32768 up, so a test that fixes its port in that range + * meets one every so often, once enough connections have come and gone. + * + * @param port The port + * @return true when none of the protocols refuses it + */ +inline bool port_is_free(uint16_t port) { + for (const int protocol : {IPPROTO_TCP, IPPROTO_SCTP}) { + const int fd = ::socket(AF_INET, SOCK_STREAM, protocol); + if (fd < 0) { + continue; // the protocol is not there, so nothing of it holds the port + } + const int one = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_ANY); + address.sin_port = htons(port); + const bool free = ::bind(fd, reinterpret_cast(&address), sizeof(address)) == 0; + ::close(fd); + if (!free) { + return false; + } + } + return true; +} + +/** + * @brief Reserve @p count consecutive ports for listeners, free now and kept from every other process. + * + * Ports come from 20000 to 31999, below the range of source ports. Being free when asked is not + * enough, because a test asks for its port some time before it listens, and another test that + * asks in between is told the same port is free. So each port is also claimed with a lock on a file + * in /tmp/libe3-test-ports, held until the process exits: no two of these tests ever take the same + * port, whatever order they ask and listen in. The search starts at a place that depends on the + * process, so that tests CTest runs side by side seldom look at the same ports at all. + * + * @param count How many consecutive ports (3 for the triple of a ZeroMQ configuration) + * @return The first of them + */ +inline uint16_t free_ports(unsigned count = 1) { + static std::atomic counter{0}; + static std::vector held; // the lock files, open until the process ends + const unsigned span = 12000; + const unsigned start = (static_cast(::getpid()) % 750) * 16; + mkdir("/tmp/libe3-test-ports", 0777); + chmod("/tmp/libe3-test-ports", 0777); + for (unsigned attempt = 0; attempt < span; ++attempt) { + const unsigned first = (start + counter.fetch_add(1)) % (span - count); + std::vector taken; + bool all = true; + for (unsigned i = 0; i < count && all; ++i) { + const uint16_t port = static_cast(20000 + first + i); + const std::string name = "/tmp/libe3-test-ports/" + std::to_string(port); + const int lock = ::open(name.c_str(), O_CREAT | O_RDWR, 0666); + if (lock < 0 || ::flock(lock, LOCK_EX | LOCK_NB) != 0 || !port_is_free(port)) { + if (lock >= 0) ::close(lock); + all = false; + } else { + taken.push_back(lock); + } + } + if (all) { + held.insert(held.end(), taken.begin(), taken.end()); + return static_cast(20000 + first); + } + for (const int lock : taken) ::close(lock); + } + throw std::runtime_error("no free port to listen on"); +} + +/** @brief One port: see free_ports(). @return The port */ +inline uint16_t free_port() { return free_ports(1); } + +inline bool uring_required() { + const char* require = std::getenv("LIBE3_REQUIRE_URING"); + return require != nullptr && *require != '\0' && *require != '0'; +} + +/** @brief Whether this build and this kernel can create an io_uring connector. */ +inline bool uring_available() { + static const bool available = [] { + auto probe = libe3::create_connector(libe3::E3LinkLayer::URING, libe3::E3TransportLayer::TCP, + "", "", "", 1, 0, 0, 1, libe3::E3Role::DAPP); + return probe != nullptr; + }(); + return available; +} + +/** + * @brief For one test that needs io_uring: true when it can run here. + * + * Otherwise prints why and returns false, so the test returns early; with + * LIBE3_REQUIRE_URING set it throws instead, which fails the test. + */ +inline bool uring_runs_here() { + if (uring_available()) { + return true; + } + if (uring_required()) { + throw std::runtime_error("io_uring is required (LIBE3_REQUIRE_URING) and cannot be used here"); + } + std::cout << "URING NOT RUN: this build or this kernel cannot create an io_uring ring " + "(built without liburing, seccomp, kernel.io_uring_disabled, or an old kernel)\n"; + return false; +} + +inline libe3::E3LinkLayer parse_link_or_throw(const char* text) { + if (std::strcmp(text, "posix") == 0) return libe3::E3LinkLayer::POSIX; + if (std::strcmp(text, "uring") == 0) return libe3::E3LinkLayer::URING; + throw std::invalid_argument(std::string("unknown link '") + text + "', expected posix or uring"); +} + +/** + * @brief Read the links from the command line. + * + * @return -1 to go on and run the tests, or the exit code main() should return: + * 77 when io_uring was asked for and cannot run here (1 with LIBE3_REQUIRE_URING), + * 2 on a bad argument + */ +inline int select_links(int argc, char** argv) { + auto& links = links_under_test(); + try { + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (i + 1 >= argc) throw std::invalid_argument("missing value after " + arg); + if (arg == "--link") { + links.ran = links.dapp = parse_link_or_throw(argv[++i]); + } else if (arg == "--ran-link") { + links.ran = parse_link_or_throw(argv[++i]); + } else if (arg == "--dapp-link") { + links.dapp = parse_link_or_throw(argv[++i]); + } else { + throw std::invalid_argument("unknown argument " + arg); + } + } + } catch (const std::exception& e) { + std::cerr << e.what() << "\n"; + return 2; + } + std::cout << "Links under test: RAN " << libe3::link_layer_to_string(links.ran) << ", dApp " + << libe3::link_layer_to_string(links.dapp) << "\n"; + const bool wants_uring = links.ran == libe3::E3LinkLayer::URING || links.dapp == libe3::E3LinkLayer::URING; + if (wants_uring && !uring_available()) { + if (uring_required()) { + std::cerr << "FAIL: io_uring is required (LIBE3_REQUIRE_URING) and cannot be used here\n"; + return 1; + } + std::cout << "URING NOT RUN: this build or this kernel cannot create an io_uring ring " + "(built without liburing, seccomp, kernel.io_uring_disabled, or an old kernel)\n"; + return 77; + } + return -1; +} + +} // namespace libe3_test + +#endif // LIBE3_TEST_LINK_UNDER_TEST_HPP diff --git a/tests/test_bench_matrix.py b/tests/test_bench_matrix.py new file mode 100644 index 00000000..fcbcbfcd --- /dev/null +++ b/tests/test_bench_matrix.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University +# SPDX-License-Identifier: Apache-2.0 +"""Checks of tools/bench_matrix.py on canned benchmark output, without running the benchmark. + + test_bench_matrix.py TOOL + +Properties verified: + - every number the comparison uses is read from the right row and column of the 18-phase table, + and the CPU line is summed; + - a wire hop below zero (the receiver was quicker than the sender's own stamp) is read as it is; + - output without a Total row is a failed run, not a zero; + - the fastest link is named, its lead over POSIX is given, and two links whose rounds overlap + are said to be within the noise; + - a cell that was not run (ZeroMQ over SCTP) has no row. +""" +import importlib.util +import sys +import unittest + +TOOL = sys.argv.pop(1) if len(sys.argv) > 1 else "tools/bench_matrix.py" +spec = importlib.util.spec_from_file_location("bench_matrix", TOOL) +bench_matrix = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bench_matrix) + + +def table(total_p50, total_p99, wire6, wire15, send5, send14, cpu_user=40.0, cpu_sys=60.0): + def row(num, p50, p99=0.0): + return f"| {num:>5} | x | `A` to `B` | 1.00 | {p50:7.2f} | {p99:7.2f} | 9.00 |" + lines = ["## Full-loop latency benchmark (N=1000 after 50 warmup)", "", + "| # | Description | Tags | mean | p50 | p99 | max |", "|---|---|---|---|---|---|---|"] + for n in range(1, 19): + value = {5: send5, 6: wire6, 14: send14, 15: wire15}.get(n, 0.5) + lines.append(row(str(n), value)) + lines.append(row("Total", total_p50, total_p99)) + lines.append("") + lines.append(f"CPU per round trip, both ends: {cpu_user:.2f} us user, {cpu_sys:.2f} us system; " + "context switches per round trip: 5.00 voluntary, 0.00 involuntary.") + return "\n".join(lines) + + +class Parse(unittest.TestCase): + def test_reads_each_number_from_its_row_and_column(self): + got = bench_matrix.parse(table(100.0, 150.0, 19.0, 20.0, 6.0, 6.5, 40.0, 60.0)) + self.assertEqual(got["rt_p50"], 100.0) + self.assertEqual(got["rt_p99"], 150.0) + self.assertEqual(got["wire_ind_p50"], 19.0) + self.assertEqual(got["wire_ctrl_p50"], 20.0) + self.assertEqual(got["send_ind_p50"], 6.0) + self.assertEqual(got["send_ctrl_p50"], 6.5) + self.assertEqual(got["cpu_us"], 100.0) + self.assertEqual(got["samples"], 1000.0) + + def test_a_wire_hop_below_zero_is_read_as_it_is(self): + # A receiver that polls can see the PDU before the sender has stamped its send as done. + got = bench_matrix.parse(table(40.0, 90.0, -0.45, -1.20, 6.0, 6.5)) + self.assertEqual(got["wire_ind_p50"], -0.45) + self.assertEqual(got["wire_ctrl_p50"], -1.20) + + def test_output_without_a_total_is_a_failed_run(self): + self.assertIsNone(bench_matrix.parse("dapp setup failed\n")) + + +class Compare(unittest.TestCase): + class Args: + rounds = 3 + iterations = 1000 + warmup = 50 + + def cells(self, values): + return {key: {"ok": [{"rt_p50": v, "rt_p99": v * 2} for v in runs], "failed": []} + for key, runs in values.items()} + + def test_names_the_fastest_and_its_lead_over_posix(self): + cells = self.cells({("posix", "ipc", "asn1"): [100, 101, 102], + ("uring-busy", "ipc", "asn1"): [70, 71, 72]}) + text = bench_matrix.compare(cells, ["posix", "uring-busy"], ["ipc"], ["asn1"], self.Args) + self.assertIn("fastest is **uring-busy** (71.0 us)", text) + self.assertIn("Against posix (101.0 us): 30% less", text) + self.assertNotIn("within the noise", text) + + def test_overlapping_rounds_are_within_the_noise(self): + cells = self.cells({("posix", "ipc", "asn1"): [100, 104, 108], + ("uring", "ipc", "asn1"): [99, 103, 109]}) + text = bench_matrix.compare(cells, ["posix", "uring"], ["ipc"], ["asn1"], self.Args) + self.assertIn("within the noise", text) + + def test_a_cell_that_was_not_run_has_no_row(self): + cells = self.cells({("posix", "sctp", "asn1"): [100, 100, 100]}) + text = bench_matrix.compare(cells, ["zmq", "posix"], ["sctp"], ["asn1"], self.Args) + self.assertNotIn("| sctp | asn1 | zmq |", text) + self.assertIn("| sctp | asn1 | posix |", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dapp_bound_to_connection_posix.cpp b/tests/test_dapp_bound_to_connection_posix.cpp index a2832c51..5d0e5194 100644 --- a/tests/test_dapp_bound_to_connection_posix.cpp +++ b/tests/test_dapp_bound_to_connection_posix.cpp @@ -3,7 +3,8 @@ * @brief A POSIX RAN acts on a message that names a dApp only if it came on that dApp's connection. * * A real RAN E3Agent and a real dApp E3Agent over POSIX IPC, and a second, - * bare connection that registers as another dApp. Properties verified: + * bare connection that registers as another dApp. CTest runs it again over + * the io_uring link and mixed (link_under_test.hpp). Properties verified: * (1) a release that names the dApp but comes on another connection is * dropped: the dApp keeps its registration, its subscription and its * RAN function, and is told nothing; @@ -16,6 +17,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include "libe3/libe3.hpp" #include "libe3/e3_agent.hpp" #include "libe3/e3_connector.hpp" @@ -102,7 +104,7 @@ struct Endpoints { E3Config make_config(E3Role role, const Endpoints& ep) { E3Config c; c.role = role; - c.link_layer = E3LinkLayer::POSIX; + c.link_layer = libe3_test::link_for(role); c.transport_layer = E3TransportLayer::IPC; c.setup_endpoint = ep.setup; c.subscriber_endpoint = ep.sub; @@ -121,7 +123,7 @@ E3Config make_config(E3Role role, const Endpoints& ep) { struct OtherConnection { explicit OtherConnection(const Endpoints& ep) : encoder(create_encoder(pick_encoding())), - connector(create_connector(E3LinkLayer::POSIX, E3TransportLayer::IPC, ep.setup, ep.sub, + connector(create_connector(libe3_test::dapp_link(), E3TransportLayer::IPC, ep.setup, ep.sub, ep.pub, 0, 0, 0, 1, E3Role::DAPP)) {} bool open() { @@ -291,6 +293,7 @@ TEST(DAppBound_posix_aControlAndAReportFromAnotherConnectionAreDropped) { ran.stop(); } -int main() { +int main(int argc, char** argv) { + if (const int rc = libe3_test::select_links(argc, argv); rc >= 0) return rc; return RUN_ALL_TESTS(); } diff --git a/tests/test_dapp_link_closed_posix.cpp b/tests/test_dapp_link_closed_posix.cpp index baaf2a46..61f959be 100644 --- a/tests/test_dapp_link_closed_posix.cpp +++ b/tests/test_dapp_link_closed_posix.cpp @@ -4,6 +4,7 @@ * * A real RAN E3Agent and real dApp E3Agents over POSIX IPC. A dApp that is * stopped without release() closes its sockets, which is all a crash does. + * CTest runs it again over the io_uring link and mixed (link_under_test.hpp). * Properties verified: * (1) the RAN removes it at once, with its idle limit off, so it is the * closed connection and not the idle time that does it: no registration, @@ -22,6 +23,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include "libe3/libe3.hpp" #include "libe3/e3_agent.hpp" #include "libe3/types.hpp" @@ -105,7 +107,7 @@ struct Endpoints { E3Config make_config(E3Role role, const Endpoints& ep, uint32_t idle_s) { E3Config c; c.role = role; - c.link_layer = E3LinkLayer::POSIX; + c.link_layer = libe3_test::link_for(role); c.transport_layer = E3TransportLayer::IPC; c.setup_endpoint = ep.setup; c.subscriber_endpoint = ep.sub; @@ -241,6 +243,7 @@ TEST(DAppLinkClosed_posix_aDAppThatReleasedIsNotRemovedAgainAndAnIdIsNotTakenOve next->stop(); } -int main() { +int main(int argc, char** argv) { + if (const int rc = libe3_test::select_links(argc, argv); rc >= 0) return rc; return RUN_ALL_TESTS(); } diff --git a/tests/test_dapp_ran_loss.cpp b/tests/test_dapp_ran_loss.cpp index 500c8655..b883bc13 100644 --- a/tests/test_dapp_ran_loss.cpp +++ b/tests/test_dapp_ran_loss.cpp @@ -21,6 +21,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include "libe3/libe3.hpp" #include "libe3/e3_agent.hpp" #include "libe3/e3_encoder.hpp" @@ -279,6 +280,11 @@ TEST(DAppRanLoss_posix_stoppedRanReportsConnectionLost) { run_ran_loss(E3LinkLayer::POSIX); } +TEST(DAppRanLoss_uring_stoppedRanReportsConnectionLost) { + if (!libe3_test::uring_runs_here()) return; + run_ran_loss(E3LinkLayer::URING); +} + #if LIBE3_HAS_ZMQ && defined(LIBE3_ENABLE_JSON) TEST(DAppRelease_json_onlyOwnReleaseEndsSession) { run_release(EncodingFormat::JSON); diff --git a/tests/test_dapp_reconnect.cpp b/tests/test_dapp_reconnect.cpp index 30afee15..e909e9ac 100644 --- a/tests/test_dapp_reconnect.cpp +++ b/tests/test_dapp_reconnect.cpp @@ -19,6 +19,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include "libe3/libe3.hpp" #include "libe3/e3_agent.hpp" #include "libe3/types.hpp" @@ -333,6 +334,11 @@ TEST(DAppReconnect_posix_registersWithANewRanByItself) { run_ran_restart(E3LinkLayer::POSIX); } +TEST(DAppReconnect_uring_registersWithANewRanByItself) { + if (!libe3_test::uring_runs_here()) return; + run_ran_restart(E3LinkLayer::URING); +} + int main() { return RUN_ALL_TESTS(); } diff --git a/tests/test_dapp_refused_setup_reconnect.cpp b/tests/test_dapp_refused_setup_reconnect.cpp index 8c475070..6d5e8786 100644 --- a/tests/test_dapp_refused_setup_reconnect.cpp +++ b/tests/test_dapp_refused_setup_reconnect.cpp @@ -16,6 +16,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include "libe3/libe3.hpp" #include "libe3/e3_agent.hpp" #include "libe3/e3_connector.hpp" @@ -54,10 +55,7 @@ EncodingFormat pick_encoding() { #endif } -uint16_t unique_port() { - static std::atomic next{0}; - return static_cast(34000 + (getpid() % 2000) * 8 + (next.fetch_add(1) % 8)); -} +uint16_t unique_port() { return libe3_test::free_port(); } std::string make_tmpdir() { char tmpl[] = "/tmp/libe3_refused_XXXXXX"; @@ -253,6 +251,17 @@ TEST(refused_setup_is_retried_posix_tcp) { run({E3LinkLayer::POSIX, E3TransportLayer::TCP, "", unique_port()}); } +TEST(refused_setup_is_retried_uring_ipc) { + if (!libe3_test::uring_runs_here()) return; + mkdir("/tmp/dapps", 0777); + run({E3LinkLayer::URING, E3TransportLayer::IPC, "ipc://" + make_tmpdir() + "/e3", 0}); +} + +TEST(refused_setup_is_retried_uring_tcp) { + if (!libe3_test::uring_runs_here()) return; + run({E3LinkLayer::URING, E3TransportLayer::TCP, "", unique_port()}); +} + #if LIBE3_HAS_ZMQ TEST(refused_setup_is_retried_zmq_ipc) { mkdir("/tmp/dapps", 0777); diff --git a/tests/test_link_availability.cpp b/tests/test_link_availability.cpp new file mode 100644 index 00000000..e3517cff --- /dev/null +++ b/tests/test_link_availability.cpp @@ -0,0 +1,120 @@ +/** + * @file test_link_availability.cpp + * @brief Each link layer is there exactly when the build has it, and the C API reaches all three. + * + * Built in every configuration, so the builds without ZeroMQ and without liburing run it too. + * Properties verified: + * (1) the factory makes a ZeroMQ connector exactly when LIBE3_HAS_ZMQ is 1, and the POSIX one always; + * (2) without liburing, asking for the io_uring link gives no connector (the factory logs why) + * and an E3Agent configured with it fails to initialise instead of using another link; + * (3) with liburing, the factory makes an io_uring connector whose link and transport are the + * ones asked for, unless the kernel refuses io_uring here (then it says so); + * (4) link_layer 2 in e3_config_t is the io_uring link: a RAN made from C with it serves an + * io_uring dApp over TCP. + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "test_framework.hpp" +#include "link_under_test.hpp" +#include "libe3/libe3.hpp" +#include "libe3/c_api.h" +#include "libe3/e3_agent.hpp" +#include "libe3/e3_connector.hpp" +#include "libe3/types.hpp" + +#include +#include +#include +#include + +using namespace libe3; +using namespace std::chrono_literals; + +namespace { + +uint16_t unique_port() { return libe3_test::free_port(); } + +std::unique_ptr make(E3LinkLayer link, E3TransportLayer transport) { + return create_connector(link, transport, "", "", "", 1, 2, 3, 1, E3Role::DAPP); +} + +#if defined(LIBE3_ENABLE_JSON) +constexpr int C_ENCODING = 1; +constexpr EncodingFormat CPP_ENCODING = EncodingFormat::JSON; +#else +constexpr int C_ENCODING = 0; +constexpr EncodingFormat CPP_ENCODING = EncodingFormat::ASN1; +#endif + +} // namespace + +TEST(the_zmq_and_posix_links_are_there_when_built) { + ASSERT_TRUE(make(E3LinkLayer::POSIX, E3TransportLayer::TCP) != nullptr); +#if LIBE3_HAS_ZMQ + ASSERT_TRUE(make(E3LinkLayer::ZMQ, E3TransportLayer::TCP) != nullptr); +#else + ASSERT_TRUE(make(E3LinkLayer::ZMQ, E3TransportLayer::TCP) == nullptr); +#endif +} + +TEST(the_uring_link_is_there_exactly_when_built_and_allowed) { +#if LIBE3_HAS_URING + if (!libe3_test::uring_runs_here()) return; + for (auto transport : {E3TransportLayer::TCP, E3TransportLayer::IPC, E3TransportLayer::SCTP}) { + auto conn = make(E3LinkLayer::URING, transport); + ASSERT_TRUE(conn != nullptr); + ASSERT_TRUE(conn->link_layer() == E3LinkLayer::URING); + ASSERT_TRUE(conn->transport_layer() == transport); + ASSERT_FALSE(conn->is_connected()); + conn->dispose(); + conn->dispose(); // twice is harmless + } +#else + ASSERT_TRUE(make(E3LinkLayer::URING, E3TransportLayer::TCP) == nullptr); + E3Config config; + config.link_layer = E3LinkLayer::URING; + config.log_level = 0; + E3Agent agent(config); + ASSERT_TRUE(agent.init() != ErrorCode::SUCCESS); // no silent fallback to another link +#endif +} + +TEST(link_layer_two_in_the_c_api_is_the_uring_link) { + if (!libe3_test::uring_runs_here()) return; + const uint16_t port = unique_port(); + e3_config_t cfg{}; + cfg.ran_identifier = "capi-uring-test"; + cfg.link_layer = 2; // io_uring + cfg.transport_layer = 1; // TCP + cfg.setup_port = port; + cfg.encoding = C_ENCODING; + cfg.log_level = 0; + e3_agent_handle_t* ran = e3_agent_create_with_config(&cfg); + ASSERT_TRUE(ran != nullptr); + ASSERT_EQ(e3_agent_init(ran), 0); + ASSERT_EQ(e3_agent_start(ran), 0); + + // A ZeroMQ RAN (what an ignored link_layer would leave) does not frame a setup request + // the way an io_uring dApp sends it, so the setup below would time out. + E3Config dapp_cfg; + dapp_cfg.role = E3Role::DAPP; + dapp_cfg.link_layer = E3LinkLayer::URING; + dapp_cfg.transport_layer = E3TransportLayer::TCP; + dapp_cfg.setup_port = port; + dapp_cfg.encoding = CPP_ENCODING; + dapp_cfg.log_level = 0; + dapp_cfg.reconnect_interval_ms = 0; + E3Agent dapp(dapp_cfg); + ASSERT_TRUE(dapp.start() == ErrorCode::SUCCESS); + const bool set_up = dapp.wait_for_setup(10000ms) == ErrorCode::SUCCESS && dapp.dapp_id().has_value(); + dapp.stop(); + e3_agent_stop(ran); + e3_agent_destroy(ran); + ASSERT_TRUE(set_up); +} + +int main() { + return RUN_ALL_TESTS(); +} diff --git a/tests/test_role_aware_connector_posix.cpp b/tests/test_role_aware_connector_posix.cpp index d61065d4..19322798 100644 --- a/tests/test_role_aware_connector_posix.cpp +++ b/tests/test_role_aware_connector_posix.cpp @@ -6,6 +6,9 @@ * produce bytes. Covers IPC (a sequenced-packet socket) and TCP. SCTP is covered in * test_role_pair_sctp, which needs the sctp kernel module. * + * The io_uring link speaks the same wire, so CTest runs this binary again with + * --link uring, and with a RAN of one link and a dApp of the other (link_under_test.hpp). + * * Properties verified: * - setup request and response round trip on the one endpoint, IPC and TCP; * - a final message sent just before a clean close is delivered, not lost; @@ -22,6 +25,7 @@ */ #include "test_framework.hpp" +#include "link_under_test.hpp" #include "libe3/types.hpp" #include "libe3/e3_connector.hpp" @@ -55,15 +59,12 @@ std::string make_tmpdir() { return std::string(d); } -// One port per test, distinct across tests and across concurrent runs. -uint16_t unique_port() { - static std::atomic next{0}; - return static_cast(28000 + (getpid() % 2000) * 8 + (next.fetch_add(1) % 8)); -} +// One port per test: free when asked for, distinct across tests and across concurrent runs. +uint16_t unique_port() { return libe3_test::free_port(); } std::unique_ptr make(E3TransportLayer transport, const std::string& endpoint, uint16_t port, E3Role role, const PosixOptions& options = PosixOptions{}) { - return create_connector(E3LinkLayer::POSIX, transport, endpoint, "", "", port, 0, 0, 1, role, options); + return create_connector(libe3_test::link_for(role), transport, endpoint, "", "", port, 0, 0, 1, role, options); } std::vector bytes(const std::string& text) { @@ -507,6 +508,7 @@ TEST(posix_tcp_over_ipv6) { ran->dispose(); } -int main() { +int main(int argc, char** argv) { + if (const int rc = libe3_test::select_links(argc, argv); rc >= 0) return rc; return RUN_ALL_TESTS(); } diff --git a/tests/test_uring_connector.cpp b/tests/test_uring_connector.cpp new file mode 100644 index 00000000..74adc3d6 --- /dev/null +++ b/tests/test_uring_connector.cpp @@ -0,0 +1,540 @@ +/** + * @file test_uring_connector.cpp + * @brief What only the io_uring link does: its ring options, batched sends, and how it stops. + * + * The wire it shares with the POSIX link is tested by running the POSIX tests with --link uring + * (test_role_aware_connector_posix and the others listed in cmake/libe3Tests.cmake). Here the + * connector is driven directly. Properties verified: + * - every ring variant (plain, registered files, registered buffers, a kernel submission + * thread, busy polling, and all of them) carries a setup and both directions on IPC, TCP and + * SCTP, and the variant asked for is the one in effect where the kernel allows it; + * - a broadcast to more peers than the ring has entries reaches every peer, once and in order; + * - a peer that stops reading does not delay what the others receive: they have each PDU + * before the send to the stalled one gives up, and the stalled one is then closed; + * - a peer that resets its connection is dropped alone and reported closed once; + * - thousands of PDUs of every size up to the limit arrive whole and in order, both ways; + * - a PDU of the limit passes and one byte more is refused before it is sent; + * - receive() waits about its half second and returns 0; shutdown() wakes a receive() that + * is waiting, a send after it is CANCELLED, and dispose() can be called twice. + * + * Needs a kernel that lets this process use io_uring; otherwise exits 77 (skipped) unless + * LIBE3_REQUIRE_URING is set. + * + * SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "test_framework.hpp" +#include "link_under_test.hpp" +#include "libe3/e3_connector.hpp" +#include "libe3/types.hpp" +#include "connector/uring_connector.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace libe3; +using namespace std::chrono_literals; +using clk = std::chrono::steady_clock; + +namespace { + +constexpr size_t LIMIT = MAX_PROTOCOL_DATA_SIZE * 2; + +std::string make_tmpdir() { + char tmpl[] = "/tmp/libe3_uring_XXXXXX"; + char* d = mkdtemp(tmpl); + if (!d) throw std::runtime_error("mkdtemp failed"); + chmod(d, 0777); + return std::string(d); +} + +uint16_t unique_port() { return libe3_test::free_port(); } + +bool sctp_here() { +#ifdef LIBE3_ENABLE_SCTP + int probe = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); + if (probe < 0) return false; + close(probe); + return true; +#else + return false; +#endif +} + +/// Sets the ring options for the connectors made while it lives; the connector reads them once. +class RingOptions { +public: + explicit RingOptions(std::map values) : values_(std::move(values)) { + for (const auto& [name, value] : values_) setenv(name.c_str(), value.c_str(), 1); + } + ~RingOptions() { + for (const auto& [name, value] : values_) unsetenv(name.c_str()); + } + +private: + std::map values_; +}; + +struct Endpoint { + E3TransportLayer transport; + std::string uri; + uint16_t port; +}; + +Endpoint endpoint_for(E3TransportLayer transport) { + if (transport == E3TransportLayer::IPC) return {transport, "ipc://" + make_tmpdir() + "/e3", 0}; + return {transport, "", unique_port()}; +} + +std::unique_ptr make(const Endpoint& where, E3Role role) { + return create_connector(E3LinkLayer::URING, where.transport, where.uri, "", "", where.port, 0, 0, 1, role); +} + +std::vector bytes(const std::string& text) { return {text.begin(), text.end()}; } +std::string text(const std::vector& data) { return {data.begin(), data.end()}; } + +std::vector receive_pdu(E3Connector& ran, int tries = 10) { + for (int i = 0; i < tries; ++i) { + std::vector buf; + if (ran.receive(buf) > 0) return buf; + } + return {}; +} + +std::vector dapp_receive(E3Connector& dapp, int tries = 4) { + for (int i = 0; i < tries; ++i) { + std::vector buf; + const int n = dapp.receive(buf); + if (n > 0) return buf; + if (n < 0) return {}; + } + return {}; +} + +std::vector setup_request(E3Connector& ran, int tries = 10) { + for (int i = 0; i < tries; ++i) { + std::vector buf; + if (ran.recv_setup_request(buf) > 0) return buf; + } + return {}; +} + +std::unique_ptr registered_dapp(E3Connector& ran, const Endpoint& where, uint64_t* token = nullptr) { + auto dapp = make(where, E3Role::DAPP); + if (!dapp || dapp->setup_initial_connection_client() != ErrorCode::SUCCESS) return nullptr; + if (dapp->send_setup_request_client(bytes("hello")) != ErrorCode::SUCCESS) return nullptr; + if (setup_request(ran).empty()) return nullptr; + if (token) *token = ran.last_setup_peer_token(); + if (ran.send_response(bytes("OK")) != ErrorCode::SUCCESS) return nullptr; + std::vector response; + int n = 0; + for (int i = 0; i < 10 && n == 0; ++i) n = dapp->recv_setup_response_client(response); + return n == 2 ? std::move(dapp) : nullptr; +} + +std::vector patterned(size_t size, uint32_t seed) { + std::vector out(size); + for (size_t i = 0; i < size; ++i) out[i] = static_cast((i * 131 + seed * 7 + (i >> 8)) & 0xFF); + return out; +} + +/// Setup, then each direction, then a broadcast to two dApps, on one transport. +void exchange_on(E3TransportLayer transport) { + const Endpoint where = endpoint_for(transport); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran != nullptr); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + uint64_t first_token = 0; + auto first = registered_dapp(*ran, where, &first_token); + auto second = registered_dapp(*ran, where); + ASSERT_TRUE(first != nullptr); + ASSERT_TRUE(second != nullptr); + + ASSERT_TRUE(first->send(bytes("from-first")) == ErrorCode::SUCCESS); + ASSERT_EQ(text(receive_pdu(*ran)), std::string("from-first")); + ASSERT_EQ(ran->last_peer_token(), first_token); + ASSERT_TRUE(ran->send_to(first_token, bytes("to-first"), PduClass::INDICATION) == ErrorCode::SUCCESS); + ASSERT_EQ(text(dapp_receive(*first)), std::string("to-first")); + ASSERT_TRUE(ran->send(bytes("to-all")) == ErrorCode::SUCCESS); + ASSERT_EQ(text(dapp_receive(*first)), std::string("to-all")); + ASSERT_EQ(text(dapp_receive(*second)), std::string("to-all")); + + second->dispose(); + first->dispose(); + ran->dispose(); +} + +void variant(const std::map& options, + const std::function& expect) { + RingOptions set(options); + auto probe = make({E3TransportLayer::TCP, "", unique_port()}, E3Role::DAPP); + ASSERT_TRUE(probe != nullptr); + auto* uring = dynamic_cast(probe.get()); + ASSERT_TRUE(uring != nullptr); + std::cout << " in effect: " << uring->settings().describe() << "\n"; + expect(uring->settings()); + probe.reset(); + + exchange_on(E3TransportLayer::IPC); + exchange_on(E3TransportLayer::TCP); + if (sctp_here()) { + exchange_on(E3TransportLayer::SCTP); + } else { + std::cout << " SCTP NOT RUN: no sctp kernel module or built without it\n"; + } +} + +int raw_tcp(uint16_t port, int rcvbuf = 0) { + for (int attempt = 0; attempt < 100; ++attempt) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (rcvbuf > 0) setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)); + struct sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (connect(fd, reinterpret_cast(&addr), sizeof(addr)) == 0) return fd; + close(fd); + std::this_thread::sleep_for(20ms); + } + return -1; +} + +bool send_framed(int fd, const std::string& payload) { + const uint32_t prefix = htonl(static_cast(payload.size())); + std::vector out(sizeof(prefix) + payload.size()); + std::memcpy(out.data(), &prefix, sizeof(prefix)); + std::memcpy(out.data() + sizeof(prefix), payload.data(), payload.size()); + return ::send(fd, out.data(), out.size(), MSG_NOSIGNAL) == static_cast(out.size()); +} + +} // namespace + +TEST(uring_plain_ring_carries_every_transport) { + variant({{"LIBE3_URING_FIXED_FILES", "0"}, {"LIBE3_URING_FIXED_BUFFERS", "0"}}, + [](const UringE3Connector::Settings& s) { + ASSERT_FALSE(s.fixed_files); + ASSERT_FALSE(s.fixed_buffers); + ASSERT_FALSE(s.sqpoll); + ASSERT_EQ(s.busy_poll_us, 0u); + }); +} + +TEST(uring_registered_files_carry_every_transport) { + variant({{"LIBE3_URING_FIXED_FILES", "1"}, {"LIBE3_URING_FIXED_BUFFERS", "0"}}, + [](const UringE3Connector::Settings& s) { ASSERT_TRUE(s.fixed_files); }); +} + +TEST(uring_registered_buffers_carry_every_transport) { + variant({{"LIBE3_URING_FIXED_FILES", "0"}, {"LIBE3_URING_FIXED_BUFFERS", "1"}}, + [](const UringE3Connector::Settings& s) { ASSERT_TRUE(s.fixed_buffers); }); +} + +TEST(uring_kernel_submission_thread_carries_every_transport) { + variant({{"LIBE3_URING_SQPOLL", "1"}}, + [](const UringE3Connector::Settings& s) { ASSERT_TRUE(s.sqpoll); }); +} + +TEST(uring_busy_polling_carries_every_transport) { + variant({{"LIBE3_URING_BUSY_POLL_US", "200"}}, + [](const UringE3Connector::Settings& s) { ASSERT_EQ(s.busy_poll_us, 200u); }); +} + +TEST(uring_everything_at_once_carries_every_transport) { + variant({{"LIBE3_URING_FIXED_FILES", "1"}, {"LIBE3_URING_FIXED_BUFFERS", "1"}, + {"LIBE3_URING_SQPOLL", "1"}, {"LIBE3_URING_BUSY_POLL_US", "1000"}}, + [](const UringE3Connector::Settings& s) { + ASSERT_TRUE(s.fixed_files); + ASSERT_TRUE(s.fixed_buffers); + ASSERT_TRUE(s.sqpoll); + ASSERT_EQ(s.busy_poll_us, 1000u); + }); +} + +TEST(uring_a_ring_option_that_makes_no_sense_is_refused_with_the_default_kept) { + RingOptions set({{"LIBE3_URING_DEPTH", "banana"}, {"LIBE3_URING_BUSY_POLL_US", "-3"}}); + auto conn = make({E3TransportLayer::TCP, "", unique_port()}, E3Role::DAPP); + ASSERT_TRUE(conn != nullptr); + const auto& s = dynamic_cast(conn.get())->settings(); + ASSERT_EQ(s.depth, UringE3Connector::Settings{}.depth); + ASSERT_EQ(s.busy_poll_us, 0u); +} + +TEST(uring_broadcast_to_more_peers_than_ring_entries_reaches_each_once_in_order) { + RingOptions set({{"LIBE3_URING_DEPTH", "4"}}); + const Endpoint where = endpoint_for(E3TransportLayer::TCP); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + std::vector> dapps; + for (int i = 0; i < 11; ++i) { + auto dapp = registered_dapp(*ran, where); + ASSERT_TRUE(dapp != nullptr); + dapps.push_back(std::move(dapp)); + } + for (int round = 0; round < 3; ++round) { + ASSERT_TRUE(ran->send(bytes("round-" + std::to_string(round))) == ErrorCode::SUCCESS); + } + for (auto& dapp : dapps) { + for (int round = 0; round < 3; ++round) { + ASSERT_EQ(text(dapp_receive(*dapp)), "round-" + std::to_string(round)); + } + ASSERT_TRUE(dapp_receive(*dapp, 1).empty()); + } + for (auto& dapp : dapps) dapp->dispose(); + ran->dispose(); +} + +TEST(uring_a_stalled_peer_does_not_delay_the_others) { + const Endpoint where = endpoint_for(E3TransportLayer::TCP); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + + // First in the accept order, so a sender that went peer by peer would wait for it first. + int stalled = raw_tcp(where.port, 2048); + ASSERT_TRUE(stalled >= 0); + ASSERT_TRUE(send_framed(stalled, "SETUP")); + ASSERT_EQ(text(setup_request(*ran)), std::string("SETUP")); + const uint64_t stalled_token = ran->last_setup_peer_token(); + ASSERT_TRUE(ran->send_response(bytes("OK")) == ErrorCode::SUCCESS); + + uint64_t good_token = 0; + auto good = registered_dapp(*ran, where, &good_token); + ASSERT_TRUE(good != nullptr); + + std::atomic stop{false}; + std::atomic received{0}; + std::vector arrivals; + std::mutex arrivals_mu; + std::thread reader([&] { + while (!stop.load()) { + std::vector buf; + if (good->receive(buf) > 0) { + std::lock_guard lk(arrivals_mu); + arrivals.push_back(clk::now()); + received.fetch_add(1); + } + } + }); + + // Broadcast until a send has to give up on the stalled peer. + const std::vector chunk(16384, 0x5A); + clk::time_point slow_start{}, slow_end{}; + int sent = 0; + for (; sent < 4000; ++sent) { + const auto t0 = clk::now(); + const ErrorCode rc = ran->send(chunk); + const auto t1 = clk::now(); + ASSERT_TRUE(rc == ErrorCode::SUCCESS); // the good peer took it + if (t1 - t0 > 300ms) { + slow_start = t0; + slow_end = t1; + ++sent; + break; + } + } + ASSERT_TRUE(slow_end != clk::time_point{}); + for (int i = 0; i < 100 && received.load() < sent; ++i) std::this_thread::sleep_for(10ms); + stop.store(true); + reader.join(); + ASSERT_EQ(received.load(), sent); + { + std::lock_guard lk(arrivals_mu); + // The PDU of the send that waited for the stalled peer reached the good one long before that send returned. + ASSERT_TRUE(arrivals.back() - slow_start < 100ms); + ASSERT_TRUE(slow_end - arrivals.back() > 200ms); + } + + // The stalled peer is closed, and only it. + std::vector closed; + for (int i = 0; i < 10 && closed.empty(); ++i) { + std::vector buf; + ran->receive(buf); + closed = ran->take_closed_peers(); + } + ASSERT_EQ(closed.size(), size_t{1}); + ASSERT_EQ(closed[0], stalled_token); + ASSERT_TRUE(ran->send_to(good_token, bytes("STILL-HERE"), PduClass::PROCEDURE) == ErrorCode::SUCCESS); + ASSERT_EQ(text(dapp_receive(*good)), std::string("STILL-HERE")); + + close(stalled); + good->dispose(); + ran->dispose(); +} + +TEST(uring_a_peer_that_resets_is_dropped_alone_and_reported_once) { + const Endpoint where = endpoint_for(E3TransportLayer::TCP); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + uint64_t good_token = 0; + auto good = registered_dapp(*ran, where, &good_token); + ASSERT_TRUE(good != nullptr); + + int rude = raw_tcp(where.port); + ASSERT_TRUE(rude >= 0); + ASSERT_TRUE(send_framed(rude, "SETUP")); + ASSERT_EQ(text(setup_request(*ran)), std::string("SETUP")); + const uint64_t rude_token = ran->last_setup_peer_token(); + struct linger hard{1, 0}; + setsockopt(rude, SOL_SOCKET, SO_LINGER, &hard, sizeof(hard)); + close(rude); // a reset, not a clean close + + std::vector closed; + for (int i = 0; i < 10 && closed.empty(); ++i) { + std::vector buf; + ran->receive(buf); + closed = ran->take_closed_peers(); + } + ASSERT_EQ(closed.size(), size_t{1}); + ASSERT_EQ(closed[0], rude_token); + ASSERT_TRUE(ran->take_closed_peers().empty()); + + ASSERT_TRUE(good->send(bytes("STILL-SERVED")) == ErrorCode::SUCCESS); + ASSERT_EQ(text(receive_pdu(*ran)), std::string("STILL-SERVED")); + ASSERT_EQ(ran->last_peer_token(), good_token); + good->dispose(); + ran->dispose(); +} + +TEST(uring_thousands_of_pdus_of_every_size_arrive_whole_and_in_order) { + for (auto transport : {E3TransportLayer::TCP, E3TransportLayer::IPC, E3TransportLayer::SCTP}) { + if (transport == E3TransportLayer::SCTP && !sctp_here()) continue; + const Endpoint where = endpoint_for(transport); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + uint64_t token = 0; + auto dapp = registered_dapp(*ran, where, &token); + ASSERT_TRUE(dapp != nullptr); + + const std::vector sizes = {1, 3, 4, 5, 63, 64, 100, 1000, 4095, 4096, 16384, 32768, LIMIT - 1, LIMIT}; + constexpr size_t COUNT = 2000; + std::atomic errors{0}; + std::thread to_ran([&] { + for (size_t i = 0; i < COUNT; ++i) { + if (dapp->send(patterned(sizes[i % sizes.size()], static_cast(i))) != ErrorCode::SUCCESS) errors.fetch_add(1); + } + }); + for (size_t i = 0; i < COUNT; ++i) { + const auto got = receive_pdu(*ran); + if (got != patterned(sizes[i % sizes.size()], static_cast(i))) { + errors.fetch_add(1); + break; + } + } + to_ran.join(); + ASSERT_EQ(errors.load(), 0); + + std::thread to_dapp([&] { + for (size_t i = 0; i < COUNT; ++i) { + if (ran->send_to(token, patterned(sizes[i % sizes.size()], static_cast(i + 7)), PduClass::INDICATION) != + ErrorCode::SUCCESS) { + errors.fetch_add(1); + } + } + }); + for (size_t i = 0; i < COUNT; ++i) { + const auto got = dapp_receive(*dapp, 20); + if (got != patterned(sizes[i % sizes.size()], static_cast(i + 7))) { + errors.fetch_add(1); + break; + } + } + to_dapp.join(); + ASSERT_EQ(errors.load(), 0); + dapp->dispose(); + ran->dispose(); + } +} + +TEST(uring_a_pdu_of_the_limit_passes_and_one_byte_more_is_refused) { + const Endpoint where = endpoint_for(E3TransportLayer::IPC); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + auto dapp = registered_dapp(*ran, where); + ASSERT_TRUE(dapp != nullptr); + ASSERT_TRUE(dapp->send(std::vector(LIMIT, 'x')) == ErrorCode::SUCCESS); + ASSERT_EQ(receive_pdu(*ran).size(), LIMIT); + ASSERT_TRUE(dapp->send(std::vector(LIMIT + 1, 'x')) == ErrorCode::INVALID_PARAM); + ASSERT_TRUE(dapp->send({}) == ErrorCode::INVALID_PARAM); + ASSERT_TRUE(ran->send(std::vector(LIMIT + 1, 'x')) == ErrorCode::INVALID_PARAM); + dapp->dispose(); + ran->dispose(); +} + +TEST(uring_receive_waits_about_half_a_second_when_nothing_comes) { + const Endpoint where = endpoint_for(E3TransportLayer::TCP); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + auto dapp = registered_dapp(*ran, where); + ASSERT_TRUE(dapp != nullptr); + for (E3Connector* side : {ran.get(), dapp.get()}) { + std::vector buf; + const auto t0 = clk::now(); + ASSERT_EQ(side->receive(buf), 0); + const auto waited = clk::now() - t0; + ASSERT_TRUE(waited > 300ms); + ASSERT_TRUE(waited < 1500ms); + } + dapp->dispose(); + ran->dispose(); +} + +TEST(uring_shutdown_wakes_a_waiting_receive_and_cancels_later_sends) { + for (E3Role role : {E3Role::RAN, E3Role::DAPP}) { + const Endpoint where = endpoint_for(E3TransportLayer::IPC); + auto ran = make(where, E3Role::RAN); + ASSERT_TRUE(ran->setup_initial_connection() == ErrorCode::SUCCESS); + auto dapp = registered_dapp(*ran, where); + ASSERT_TRUE(dapp != nullptr); + E3Connector& side = role == E3Role::RAN ? *ran : *dapp; + + clk::time_point returned_at; + std::thread waiter([&] { + std::vector buf; + side.receive(buf); // waits up to half a second + returned_at = clk::now(); + }); + std::this_thread::sleep_for(100ms); + const auto t0 = clk::now(); + side.shutdown(); + side.shutdown(); // twice is harmless + waiter.join(); + ASSERT_TRUE(returned_at >= t0); + ASSERT_TRUE(returned_at - t0 < 150ms); + ASSERT_TRUE(side.send(bytes("late")) == ErrorCode::CANCELLED); + side.dispose(); + side.dispose(); + ran->dispose(); + dapp->dispose(); + } +} + +int main(int argc, char** argv) { + if (const int rc = libe3_test::select_links(argc, argv); rc >= 0) return rc; + if (!libe3_test::uring_available()) { + if (libe3_test::uring_required()) { + std::cerr << "FAIL: io_uring is required (LIBE3_REQUIRE_URING) and cannot be used here\n"; + return 1; + } + std::cout << "URING NOT RUN: this build or this kernel cannot create an io_uring ring\n"; + return 77; + } + return RUN_ALL_TESTS(); +} diff --git a/tools/bench_matrix.py b/tools/bench_matrix.py new file mode 100644 index 00000000..17505cb0 --- /dev/null +++ b/tools/bench_matrix.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 Northeastern University +# SPDX-License-Identifier: Apache-2.0 +"""Run the full-loop latency benchmark over links, transports and encodings, and compare them. + +Each cell (link variant, transport, encoding) is run --rounds times. The rounds are interleaved +(round 1 of every cell, then round 2 of every cell, ...), so a drift of the machine (heat, other +tenants) hits every cell alike instead of the one that happened to run last. Each run prints the +18-phase table of test_bench_full_loop_latency; this script keeps, per run, the p50 and p99 of +the round trip, the p50 of the send calls (phases 5 and 14), of the two wire hops (6 and 15) and +of the two receive decodes, and the CPU the process spent per round trip. + +A cell's number is the median over its rounds, with the range of the rounds beside it. The +fastest link of a transport is named only when its slowest round beats the best round of every +other link; otherwise the table says the leaders are within the run-to-run noise. + + tools/bench_matrix.py --bench build/test_bench_full_loop_latency \\ + --links zmq,posix,uring,uring-busy --transports ipc,tcp,sctp --rounds 5 \\ + --out-md comparison.md --out-csv runs.csv + +The link variants are the io_uring link with ring options set through LIBE3_URING_* (see +docs/connectors/uring.md): uring (the defaults), uring-plain (no registered files), uring-fixedbuf +(registered buffers), uring-sqpoll (a kernel submission thread), uring-busy (busy polling for up +to --busy-us), uring-sqpoll-busy (both). +""" +from __future__ import annotations + +import argparse +import csv +import os +import re +import statistics +import subprocess +import sys +import time + +VARIANTS = { + "zmq": ("zmq", {}), + "posix": ("posix", {}), + "uring": ("uring", {}), + "uring-plain": ("uring", {"LIBE3_URING_FIXED_FILES": "0", "LIBE3_URING_FIXED_BUFFERS": "0"}), + "uring-fixedbuf": ("uring", {"LIBE3_URING_FIXED_BUFFERS": "1"}), + "uring-sqpoll": ("uring", {"LIBE3_URING_SQPOLL": "1"}), + "uring-busy": ("uring", {"LIBE3_URING_BUSY_POLL_US": "{busy}"}), + "uring-sqpoll-busy": ("uring", {"LIBE3_URING_SQPOLL": "1", "LIBE3_URING_BUSY_POLL_US": "{busy}"}), +} + +ROW = re.compile(r"^\|\s*(\w+)\s*\|.*\|\s*(-?[\d.]+)\s*\|\s*(-?[\d.]+)\s*\|\s*(-?[\d.]+)\s*\|\s*(-?[\d.]+)\s*\|\s*$") +CPU = re.compile(r"CPU per round trip, both ends: ([\d.]+) us user, ([\d.]+) us system; " + r"context switches per round trip: ([\d.]+) voluntary, ([\d.]+) involuntary") +SAMPLES = re.compile(r"\(N=(\d+) after") + +# (column, table row, statistic): statistic is the column index of mean/p50/p99/max. +METRICS = [ + ("rt_p50", "Total", 1), + ("rt_p99", "Total", 2), + ("send_ind_p50", "5", 1), + ("wire_ind_p50", "6", 1), + ("send_ctrl_p50", "14", 1), + ("wire_ctrl_p50", "15", 1), +] + + +def parse(output: str) -> dict[str, float] | None: + rows = {} + for line in output.splitlines(): + m = ROW.match(line) + if m: + rows[m.group(1)] = [float(m.group(i)) for i in range(2, 6)] + if "Total" not in rows: + return None + out = {name: rows[row][stat] for name, row, stat in METRICS if row in rows} + m = CPU.search(output) + if m: + out["cpu_us"] = float(m.group(1)) + float(m.group(2)) + out["ctxsw"] = float(m.group(3)) + float(m.group(4)) + m = SAMPLES.search(output) + out["samples"] = float(m.group(1)) if m else 0.0 + return out + + +def run_one(args, variant: str, transport: str, encoding: str) -> tuple[dict | None, str]: + link, extra = VARIANTS[variant] + env = {k: v for k, v in os.environ.items() if not k.startswith("LIBE3_URING_")} + env.update({k: v.format(busy=args.busy_us) for k, v in extra.items()}) + cmd = [args.bench, "--link", link, "--transport", transport, "--encoding", encoding, + "--iterations", str(args.iterations), "--warmup", str(args.warmup)] + if args.ran_cpu is not None: + cmd += ["--ran-cpu", str(args.ran_cpu)] + if args.dapp_cpu is not None: + cmd += ["--dapp-cpu", str(args.dapp_cpu)] + try: + proc = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=args.timeout) + except subprocess.TimeoutExpired: + return None, "timeout" + result = parse(proc.stdout) + if proc.returncode != 0 or result is None: + tail = (proc.stdout + proc.stderr).strip().splitlines()[-4:] + return None, f"exit {proc.returncode}: " + " / ".join(tail) + return result, "" + + +def median(values: list[float]) -> float: + return statistics.median(values) if values else float("nan") + + +def fmt(values: list[float]) -> str: + if not values: + return "-" + mid = median(values) + if len(values) == 1: + return f"{mid:.1f}" + return f"{mid:.1f} [{min(values):.1f} to {max(values):.1f}]" + + +def compare(cells: dict, links: list[str], transports: list[str], encodings: list[str], args) -> str: + lines = [] + lines.append("## Connector comparison: full-loop round trip\n") + lines.append(f"Same machine, {args.rounds} interleaved round(s) per cell, {args.iterations} round trips " + f"per run after {args.warmup} of warm-up. Each value is the median over the rounds of " + "that round's statistic, with the range of the rounds in brackets. Microseconds. " + "CPU is user plus system time of the process (both ends) per round trip.\n") + header = ("| Transport | Encoding | Link | Round trip p50 | Round trip p99 | Wire RAN to dApp p50 " + "| Wire dApp to RAN p50 | Send indication p50 | Send control p50 | CPU per round trip | Runs |") + lines.append(header) + lines.append("|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|") + verdicts = [] + for transport in transports: + for encoding in encodings: + ranked = [] + for link in links: + runs = cells.get((link, transport, encoding)) + if runs is None: + continue # not run (ZeroMQ over SCTP) + ok = runs["ok"] + col = lambda name: [r[name] for r in ok if name in r] # noqa: E731 + if ok: + ranked.append((median(col("rt_p50")), link, col("rt_p50"))) + failed = f" ({len(runs['failed'])} failed)" if runs["failed"] else "" + lines.append( + f"| {transport} | {encoding} | {link} | {fmt(col('rt_p50'))} | {fmt(col('rt_p99'))} " + f"| {fmt(col('wire_ind_p50'))} | {fmt(col('wire_ctrl_p50'))} | {fmt(col('send_ind_p50'))} " + f"| {fmt(col('send_ctrl_p50'))} | {fmt(col('cpu_us'))} | {len(ok)}{failed} |") + if not ranked: + continue + ranked.sort() + best_mid, best_link, best_runs = ranked[0] + text = f"- **{transport}/{encoding}**: fastest is **{best_link}** ({best_mid:.1f} us)" + others = ranked[1:] + if others: + clear = all(max(best_runs) < min(runs) for _, _, runs in others) + second_mid, second_link, _ = others[0] + text += f", then {second_link} ({second_mid:.1f} us, {100 * (second_mid - best_mid) / second_mid:.0f}% slower)" + if not clear: + text += "; the leaders overlap round to round, so the order of the first two is within the noise" + posix = [(m, r) for m, link, r in ranked if link == "posix"] + if posix and best_link != "posix": + p_mid = posix[0][0] + text += f". Against posix ({p_mid:.1f} us): {100 * (p_mid - best_mid) / p_mid:.0f}% less" + verdicts.append(text + ".") + lines.append("") + lines.append("### Fastest link per transport\n") + lines.extend(verdicts) + lines.append("") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--bench", required=True, help="path of test_bench_full_loop_latency") + parser.add_argument("--links", default="zmq,posix,uring", help=f"comma list of {', '.join(VARIANTS)}") + parser.add_argument("--transports", default="ipc,tcp") + parser.add_argument("--encodings", default="asn1") + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--iterations", type=int, default=1000) + parser.add_argument("--warmup", type=int, default=50) + parser.add_argument("--busy-us", type=int, default=1000, help="busy-poll budget of the uring-*busy variants") + parser.add_argument("--ran-cpu", type=int) + parser.add_argument("--dapp-cpu", type=int) + parser.add_argument("--timeout", type=int, default=300, help="seconds one run may take") + parser.add_argument("--out-md", help="write the comparison here as well as to stdout") + parser.add_argument("--out-csv", help="write every run here") + args = parser.parse_args() + + links = [x for x in args.links.split(",") if x] + for link in links: + if link not in VARIANTS: + parser.error(f"unknown link variant {link}") + transports = [x for x in args.transports.split(",") if x] + encodings = [x for x in args.encodings.split(",") if x] + + cells: dict = {} + rows = [] + # ZeroMQ has no SCTP transport. + order = [(l, t, e) for t in transports for e in encodings for l in links + if not (VARIANTS[l][0] == "zmq" and t == "sctp")] + for rnd in range(args.rounds): + # Rotate the order each round so no cell always runs right after the same neighbour. + shift = rnd % len(order) + for link, transport, encoding in order[shift:] + order[:shift]: + started = time.time() + result, error = run_one(args, link, transport, encoding) + runs = cells.setdefault((link, transport, encoding), {"ok": [], "failed": []}) + if result is None: + runs["failed"].append(error) + print(f"round {rnd + 1} {link}/{transport}/{encoding}: FAILED {error}", file=sys.stderr) + continue + runs["ok"].append(result) + rows.append({"round": rnd + 1, "link": link, "transport": transport, "encoding": encoding, + **{k: f"{v:.3f}" for k, v in result.items()}}) + print(f"round {rnd + 1} {link}/{transport}/{encoding}: round trip p50 {result['rt_p50']:.1f} us " + f"({time.time() - started:.1f} s)", file=sys.stderr) + + report = compare(cells, links, transports, encodings, args) + print(report) + if args.out_md: + with open(args.out_md, "w", encoding="utf-8") as handle: + handle.write(report) + if args.out_csv and rows: + fields = sorted({k for r in rows for k in r}, key=lambda k: (k not in ("round", "link", "transport", "encoding"), k)) + with open(args.out_csv, "w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + failed = sum(len(r["failed"]) for r in cells.values()) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 46032717e9628e13fe91e242bc66669d6b18524c Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Wed, 11 Nov 2026 15:25:59 -0500 Subject: [PATCH 5/7] ci, docker, examples: run the io_uring link where the pipeline runs the others The E2E dApp job runs the links [zmq, posix, uring] over [ipc, tcp, sctp] (ZeroMQ has no SCTP) with SCTP built in; the unit and integration jobs set LIBE3_REQUIRE_URING=1 and print the kernel's io_uring settings first, so a runner that refuses io_uring fails instead of skipping; one job builds without liburing and checks that the link is reported as not built. The full-loop job runs tools/bench_matrix.py over ZeroMQ, POSIX, io_uring, and io_uring with busy polling with and without the kernel submission thread, on IPC, TCP and SCTP, three rounds, and appends the table to the CI report. The images carry liburing2, and run-pair.sh starts a container with seccomp unconfined when the link is uring (Docker's default profile refuses io_uring_setup). example_simple_agent and example_simple_dapp accept --link uring. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- .github/workflows/e2e-dapp-test.yml | 29 ++++++++++++++++++--------- .github/workflows/full-loop-bench.yml | 20 ++++++++++++++++-- .github/workflows/pr-tests.yml | 24 ++++++++++++++++++++++ docker/Dockerfile.bench | 2 +- docker/Dockerfile.build | 3 ++- docker/Dockerfile.dapp-sim | 2 +- docker/Dockerfile.ran-sim | 2 +- docker/README.md | 14 +++++++++++-- docker/run-pair.sh | 11 +++++++--- examples/simple_agent.cpp | 3 ++- examples/simple_dapp.cpp | 3 ++- 11 files changed, 91 insertions(+), 22 deletions(-) diff --git a/.github/workflows/e2e-dapp-test.yml b/.github/workflows/e2e-dapp-test.yml index fe216fe8..dd2314c1 100644 --- a/.github/workflows/e2e-dapp-test.yml +++ b/.github/workflows/e2e-dapp-test.yml @@ -7,8 +7,8 @@ name: E2E dApp Integration # every supported (link, transport) combination, asserts both exit cleanly, # and publishes a per-combination summary for the consolidated CI report. # -# POSIX/SCTP is excluded because ubuntu-latest runners lack the sctp kernel -# module. The code path is still built; only the CI matrix excludes it. +# The POSIX and the io_uring link run on IPC, TCP and SCTP (the hosted runner +# opens SCTP sockets; only the headers are installed here). ZeroMQ has no SCTP. on: workflow_call: @@ -16,7 +16,7 @@ on: jobs: asn1c-cache: - # Builds/warms the asn1c cache once, ahead of the 4 matrix legs below, so + # Builds/warms the asn1c cache once, ahead of the 8 matrix legs below, so # they don't race each other rebuilding it from source on a cold cache. name: Warm asn1c cache runs-on: ubuntu-latest @@ -52,11 +52,11 @@ jobs: strategy: fail-fast: false matrix: - # POSIX/SCTP is intentionally excluded — ubuntu-latest runners lack - # the sctp kernel module. The code path is built; we just don't - # exercise it in CI. - link: [zmq, posix] - transport: [ipc, tcp] + link: [zmq, posix, uring] + transport: [ipc, tcp, sctp] + exclude: + - link: zmq + transport: sctp steps: - name: Checkout repository @@ -77,11 +77,22 @@ jobs: run: | chmod +x ./build_libe3 ./build_libe3 -I + sudo apt-get install -y libsctp-dev - name: Build Release with examples run: | - cmake -S . -B build -DCMAKE_BUILD_TYPE=Release + cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DLIBE3_ENABLE_SCTP=ON cmake --build build -j $(nproc) + grep -q '^LIBE3_WITH_URING:INTERNAL=ON' build/CMakeCache.txt || { + echo "the io_uring link was not built (liburing missing?)"; exit 1; } + + # What a failing uring or sctp cell needs to be read: the kernel, and whether it lets this job use them. + - name: Kernel, io_uring and SCTP on this runner + run: | + uname -r + echo "kernel.io_uring_disabled=$(cat /proc/sys/kernel/io_uring_disabled 2>/dev/null || echo absent)" + python3 -c "import socket; socket.socket(socket.AF_INET, socket.SOCK_STREAM, 132).close(); print('SCTP sockets: yes')" \ + || echo "SCTP sockets: no" - name: Prepare IPC directory run: | diff --git a/.github/workflows/full-loop-bench.yml b/.github/workflows/full-loop-bench.yml index 74eb8d8a..91c9d866 100644 --- a/.github/workflows/full-loop-bench.yml +++ b/.github/workflows/full-loop-bench.yml @@ -6,7 +6,9 @@ name: Full-loop Latency Benchmark # Runs the full-loop control-loop latency benchmark (tests/integration/ # bench_full_loop_latency.cpp) on every PR and publishes the per-phase # markdown table for the consolidated CI report, mirroring the MPMC queue -# benchmark workflow. +# benchmark workflow. It also runs the benchmark over every link (ZeroMQ, +# POSIX, io_uring and two io_uring variants) and transport on this one runner, +# interleaved (tools/bench_matrix.py), and reports which link is fastest. on: workflow_call: @@ -39,13 +41,15 @@ jobs: run: | chmod +x ./build_libe3 ./build_libe3 -I + sudo apt-get install -y libsctp-dev - name: Build Release with integration tests run: | cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ -DLIBE3_BUILD_INTEGRATION_TESTS=ON \ - -DLIBE3_ENABLE_LATREC=ON + -DLIBE3_ENABLE_LATREC=ON \ + -DLIBE3_ENABLE_SCTP=ON cmake --build build --target test_bench_full_loop_latency -j $(nproc) - name: Run benchmark @@ -55,6 +59,14 @@ jobs: | grep -v "^\[20" > bench_results.md cat bench_results.md + - name: Compare the links + run: | + uname -r + echo "kernel.io_uring_disabled=$(cat /proc/sys/kernel/io_uring_disabled 2>/dev/null || echo absent)" + python3 tools/bench_matrix.py --bench ./build/test_bench_full_loop_latency \ + --links zmq,posix,uring,uring-busy,uring-sqpoll-busy --transports ipc,tcp,sctp \ + --rounds 3 --iterations 1000 --out-md bench_matrix.md --out-csv bench_matrix.csv + - name: Assemble report fragment if: ${{ always() }} run: | @@ -66,6 +78,10 @@ jobs: echo "" echo "_ubuntu-latest, Release build, ZMQ + IPC, ASN.1 APER._" echo "" + if [ -f bench_matrix.md ]; then + sed 's/^## /#### /; s/^### /#### /' bench_matrix.md + echo "" + fi echo "_These numbers are measured inside a GitHub Actions container and should be treated as an upper bound on E3AP's and the library's own latency, not a representative deployment measurement._" } > ci-report-full-loop.md diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 7b747e27..2084a953 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -40,6 +40,10 @@ jobs: needs: asn1c-cache runs-on: ${{ matrix.os }} timeout-minutes: 20 + # The io_uring tests skip (exit 77) where the kernel refuses io_uring; this runner is meant + # to run them, so a skip fails here instead of passing unseen. + env: + LIBE3_REQUIRE_URING: "1" strategy: matrix: os: [ubuntu-latest] @@ -64,6 +68,11 @@ jobs: chmod +x ./build_libe3 ./build_libe3 -I + - name: Kernel and io_uring on this runner + run: | + uname -r + echo "kernel.io_uring_disabled=$(cat /proc/sys/kernel/io_uring_disabled 2>/dev/null || echo absent)" + - name: Build and test with build_libe3 (${{ matrix.build_type }}) shell: bash run: | @@ -80,6 +89,8 @@ jobs: runs-on: ubuntu-latest needs: test timeout-minutes: 20 + env: + LIBE3_REQUIRE_URING: "1" steps: - name: Checkout repository uses: actions/checkout@v5 @@ -170,6 +181,19 @@ jobs: cmake --build build-nozmq -j $(nproc) cd build-nozmq && ctest --output-on-failure + # The io_uring link is optional too: without it the library must build, `--link uring` + # must fail cleanly (test_link_availability), and no test may need liburing. + - name: Configure, build and test with the io_uring link disabled + run: | + cmake -S . -B build-nouring \ + -DCMAKE_BUILD_TYPE=Release \ + -DLIBE3_ENABLE_URING=OFF + cmake --build build-nouring -j $(nproc) + if grep -q '^LIBE3_WITH_URING:INTERNAL=ON' build-nouring/CMakeCache.txt; then + echo "LIBE3_ENABLE_URING=OFF still built the io_uring link"; exit 1 + fi + cd build-nouring && ctest --output-on-failure + swig: name: SWIG Python Bindings runs-on: ubuntu-latest diff --git a/docker/Dockerfile.bench b/docker/Dockerfile.bench index 6fbb706e..edcacb9d 100644 --- a/docker/Dockerfile.bench +++ b/docker/Dockerfile.bench @@ -11,7 +11,7 @@ FROM ${BUILD_IMAGE} AS build FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ - && apt-get install -y --no-install-recommends libzmq5 libsctp1 libprotobuf32t64 \ + && apt-get install -y --no-install-recommends libzmq5 libsctp1 libprotobuf32t64 liburing2 \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p -m 0777 /tmp/dapps COPY --from=build /src/build/test_bench_full_loop_latency /usr/local/bin/test_bench_full_loop_latency diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build index 49db34df..18c3a865 100644 --- a/docker/Dockerfile.build +++ b/docker/Dockerfile.build @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 # Builds libe3, the examples, the integration tests and the latency benchmark -# with every encoding and SCTP. The other images copy their binaries from it. +# with every encoding, SCTP and the io_uring link (build_libe3 -I installs +# liburing-dev). The other images copy their binaries from it. # # docker build -f docker/Dockerfile.build -t libe3-build . diff --git a/docker/Dockerfile.dapp-sim b/docker/Dockerfile.dapp-sim index 5f604619..a77d3dd4 100644 --- a/docker/Dockerfile.dapp-sim +++ b/docker/Dockerfile.dapp-sim @@ -11,7 +11,7 @@ FROM ${BUILD_IMAGE} AS build FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ - && apt-get install -y --no-install-recommends libzmq5 libsctp1 libprotobuf32t64 \ + && apt-get install -y --no-install-recommends libzmq5 libsctp1 libprotobuf32t64 liburing2 \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p -m 0777 /tmp/dapps COPY --from=build /src/build/example_simple_dapp /usr/local/bin/example_simple_dapp diff --git a/docker/Dockerfile.ran-sim b/docker/Dockerfile.ran-sim index 5a99ef48..16e2d0e1 100644 --- a/docker/Dockerfile.ran-sim +++ b/docker/Dockerfile.ran-sim @@ -11,7 +11,7 @@ FROM ${BUILD_IMAGE} AS build FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ - && apt-get install -y --no-install-recommends libzmq5 libsctp1 libprotobuf32t64 \ + && apt-get install -y --no-install-recommends libzmq5 libsctp1 libprotobuf32t64 liburing2 \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p -m 0777 /tmp/dapps COPY --from=build /src/build/example_simple_agent /usr/local/bin/example_simple_agent diff --git a/docker/README.md b/docker/README.md index 0b5c7a8c..38babe92 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ same files serve a laptop, a CI job and a cluster pipeline. | Image | File | Runs | |---|---|---| -| `libe3-build` | `Dockerfile.build` | Nothing. It holds the source, the toolchain and the build with every encoding, SCTP, the integration tests and latrec on. The others copy from it. | +| `libe3-build` | `Dockerfile.build` | Nothing. It holds the source, the toolchain and the build with every encoding, SCTP, the io_uring link, the integration tests and latrec on. The others copy from it. | | `libe3-ran-sim` | `Dockerfile.ran-sim` | `example_simple_agent`, a RAN E3 termination with the reference Service Model | | `libe3-dapp-sim` | `Dockerfile.dapp-sim` | `example_simple_dapp`, a dApp E3 termination | | `libe3-bench` | `Dockerfile.bench` | `test_bench_full_loop_latency`, the full-loop latency benchmark | @@ -39,11 +39,21 @@ docker/run-pair.sh posix sctp asn1 network namespace, with one shared directory for IPC. It passes when the dApp exits 0 and prints at least one indication. The endpoints and ports are the library defaults; the arguments after the image name are those of the examples -(`--link zmq|posix`, `--transport ipc|tcp|sctp`, `--encoding asn1|json|protobuf`). +(`--link zmq|posix|uring`, `--transport ipc|tcp|sctp`, `--encoding asn1|json|protobuf`). SCTP needs the `sctp` kernel module on the machine that runs the containers: containers share the host kernel. +The io_uring link (`--link uring`) needs the kernel to let the container use +io_uring. Docker's default seccomp profile refuses it, so `run-pair.sh` starts +both containers with `--security-opt seccomp=unconfined` for that link; anywhere +else, run them that way or with a profile that allows `io_uring_setup`, +`io_uring_enter` and `io_uring_register`. The benchmark image needs the same: + +``` +docker run --rm --security-opt seccomp=unconfined libe3-bench:local --link uring +``` + ## Run the benchmark ``` diff --git a/docker/run-pair.sh b/docker/run-pair.sh index 90fce300..e859d87e 100755 --- a/docker/run-pair.sh +++ b/docker/run-pair.sh @@ -9,7 +9,8 @@ # docker/run-pair.sh [LINK] [TRANSPORT] [ENCODING] e.g. posix sctp asn1 # # TAG picks the image tag (default: local). SECONDS_TO_RUN is how long the dApp -# runs (default: 8). +# runs (default: 8). The uring link runs both containers with seccomp unconfined: +# Docker's default profile refuses the io_uring system calls. set -euo pipefail @@ -23,6 +24,10 @@ suffix=$$ ran=libe3-ran-$suffix ipc=libe3-ipc-$suffix args=(--link "$link" --transport "$transport" --encoding "$encoding") +security=() +if [ "$link" = uring ]; then + security=(--security-opt seccomp=unconfined) +fi cleanup() { docker rm -f "$ran" >/dev/null 2>&1 || true @@ -31,11 +36,11 @@ cleanup() { trap cleanup EXIT docker volume create "$ipc" >/dev/null -docker run -d --name "$ran" -v "$ipc:/tmp/dapps" "libe3-ran-sim:$tag" "${args[@]}" >/dev/null +docker run -d --name "$ran" "${security[@]}" -v "$ipc:/tmp/dapps" "libe3-ran-sim:$tag" "${args[@]}" >/dev/null sleep 1 set +e -dapp_log=$(timeout -k 10 $((run_for + 15)) docker run --rm --network "container:$ran" \ +dapp_log=$(timeout -k 10 $((run_for + 15)) docker run --rm --network "container:$ran" "${security[@]}" \ -v "$ipc:/tmp/dapps" "libe3-dapp-sim:$tag" \ "${args[@]}" --control --timed "$run_for" 2>&1) dapp_exit=$? diff --git a/examples/simple_agent.cpp b/examples/simple_agent.cpp index 59cd9bd8..2a2a9345 100644 --- a/examples/simple_agent.cpp +++ b/examples/simple_agent.cpp @@ -34,7 +34,7 @@ void signal_handler(int) { void print_usage(const char* program_name) { std::cout << "Usage: " << program_name << " [OPTIONS]\n\n" << "Options:\n" - << " -l, --link Link layer: zmq, posix (default: zmq)\n" + << " -l, --link Link layer: zmq, posix, uring (default: zmq)\n" << " -t, --transport Transport layer: sctp, tcp, ipc (default: ipc)\n" << " -e, --encoding Encoding format: asn1, json, protobuf (default: asn1)\n" << " -r, --ran_id RAN identifier advertised in setup (default: example-ran-001)\n" @@ -52,6 +52,7 @@ void print_usage(const char* program_name) { libe3::E3LinkLayer parse_link_layer(const char* str) { if (std::strcmp(str, "zmq") == 0) return libe3::E3LinkLayer::ZMQ; if (std::strcmp(str, "posix") == 0) return libe3::E3LinkLayer::POSIX; + if (std::strcmp(str, "uring") == 0) return libe3::E3LinkLayer::URING; std::cerr << "Invalid link layer: " << str << ". Using default (zmq).\n"; return libe3::E3LinkLayer::POSIX; } diff --git a/examples/simple_dapp.cpp b/examples/simple_dapp.cpp index 97edd861..1027ea91 100644 --- a/examples/simple_dapp.cpp +++ b/examples/simple_dapp.cpp @@ -48,7 +48,7 @@ static void signal_handler(int) { static void print_usage(const char* program_name) { std::cout << "Usage: " << program_name << " [OPTIONS]\n\n" << "Options:\n" - << " -l, --link Link layer: zmq, posix (default: zmq)\n" + << " -l, --link Link layer: zmq, posix, uring (default: zmq)\n" << " -t, --transport Transport: sctp, tcp, ipc (default: ipc)\n" << " -e, --encoding Encoding: asn1, json, protobuf (default: asn1)\n" << " -c, --control Send a Simple-Control every 5th indication\n" @@ -101,6 +101,7 @@ static std::string peer_label(const std::string& dir, int offset) { static libe3::E3LinkLayer parse_link_layer(const char* str) { if (std::strcmp(str, "zmq") == 0) return libe3::E3LinkLayer::ZMQ; if (std::strcmp(str, "posix") == 0) return libe3::E3LinkLayer::POSIX; + if (std::strcmp(str, "uring") == 0) return libe3::E3LinkLayer::URING; std::cerr << "Invalid link layer: " << str << " (using zmq)\n"; return libe3::E3LinkLayer::ZMQ; } From eb8f907d21898fe7022a43a7a028050ca4ff004d Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Wed, 11 Nov 2026 15:26:00 -0500 Subject: [PATCH 6/7] docs: the io_uring link, its decisions and its measurements docs/connectors/uring.md says what a user can do with the link, which link to pick, the architecture and the eleven decisions with what the others would have made of it, where it runs, the ring options, notes for a real-time deployment, the tests, the measurements on an x86-64 server and on an Arm virtual machine (every number is in the page and in the tools' CSV output, nothing is rounded to a story) and what was deferred with its cost. The README, CLAUDE.md, the path-A control-loop page (the wire hop can read below zero), the integrator guide (the link and its framing code pointers, the SCTP timer bound), the encoding chapter of the specification (one sentence: how a link moves the bytes is not part of E3AP; the Word edition is rebuilt), interop.md (rows of the pairs table, a mixed-links note, a ledger row for the new link and for the SCTP timer bound), and the SWIG README are updated. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- CLAUDE.md | 6 +- README.md | 13 +- docs/connectors/uring.md | 516 +++++++++++++++++++ docs/control-loops/path-a-e3-loop.md | 13 + docs/e3ap/04-encoding.md | 5 + docs/e3ap/O-RAN.XXX.TS.E3AP-R000-v00.01.docx | Bin 90426 -> 90565 bytes docs/e3ap/integrator-guide.md | 16 +- docs/interop.md | 7 +- swig/README.md | 2 +- 9 files changed, 560 insertions(+), 18 deletions(-) create mode 100644 docs/connectors/uring.md diff --git a/CLAUDE.md b/CLAUDE.md index 17b4c26d..de62aa09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What libe3 is -libe3 is a vendor-neutral C++17 library that implements the E3AP (E3 Application Protocol) on both sides of the O-RAN E3 interface: the RAN function (server) and the dApp (client). A single entry-point class backs both roles, switched by one config field. Transport (ZeroMQ or POSIX TCP/SCTP/Unix-domain IPC) and encoding (ASN.1 APER, JSON, and Protocol Buffers) are pluggable and selected at runtime. The data path is built for sub-millisecond, inference-driven control loops (lock-free queues, dedicated I/O threads, optional CPU pinning). It ships a C API and Python (SWIG) bindings. +libe3 is a vendor-neutral C++17 library that implements the E3AP (E3 Application Protocol) on both sides of the O-RAN E3 interface: the RAN function (server) and the dApp (client). A single entry-point class backs both roles, switched by one config field. Transport (ZeroMQ, or POSIX TCP/SCTP/Unix-domain IPC driven by plain system calls or by io_uring) and encoding (ASN.1 APER, JSON, and Protocol Buffers) are pluggable and selected at runtime. The data path is built for sub-millisecond, inference-driven control loops (lock-free queues, dedicated I/O threads, optional CPU pinning). It ships a C API and Python (SWIG) bindings. ## Build & test commands @@ -49,7 +49,7 @@ ctest -LE integration # only unit tests PYTHONPATH=build/swig python3 tests/test_swig_smoke.py # SWIG bindings smoke test ``` -**Build options** live in `cmake/libe3Options.cmake`: `LIBE3_BUILD_TESTS` (ON), `LIBE3_BUILD_EXAMPLES` (ON), `LIBE3_BUILD_INTEGRATION_TESTS` (OFF), `LIBE3_ENABLE_SWIG` (OFF), `LIBE3_ENABLE_ZMQ` (ON), `LIBE3_ENABLE_ASN1` (ON), `LIBE3_ENABLE_E2SM_DAPP` (ON, needs ASN1; builds the E2SM-DAPP codec libraries, see `docs/e2sm-dapp/`), `LIBE3_ENABLE_JSON` (OFF), `LIBE3_ENABLE_PROTOBUF` (OFF), `LIBE3_ENABLE_ASAN`/`LIBE3_ENABLE_TSAN` (OFF), `LIBE3_BUILD_DOCS` (OFF). **At least one of `LIBE3_ENABLE_ASN1` / `LIBE3_ENABLE_JSON` / `LIBE3_ENABLE_PROTOBUF` must be ON**; any combination can be built together and the encoder is chosen at runtime. `build_libe3` has a dedicated `--enable-protobuf` flag; JSON and SWIG use `--cmake-opt`. +**Build options** live in `cmake/libe3Options.cmake`: `LIBE3_BUILD_TESTS` (ON), `LIBE3_BUILD_EXAMPLES` (ON), `LIBE3_BUILD_INTEGRATION_TESTS` (OFF), `LIBE3_ENABLE_SWIG` (OFF), `LIBE3_ENABLE_ZMQ` (ON), `LIBE3_ENABLE_URING` (ON when Linux and liburing allow it), `LIBE3_ENABLE_ASN1` (ON), `LIBE3_ENABLE_E2SM_DAPP` (ON, needs ASN1; builds the E2SM-DAPP codec libraries, see `docs/e2sm-dapp/`), `LIBE3_ENABLE_JSON` (OFF), `LIBE3_ENABLE_PROTOBUF` (OFF), `LIBE3_ENABLE_ASAN`/`LIBE3_ENABLE_TSAN` (OFF), `LIBE3_BUILD_DOCS` (OFF). **At least one of `LIBE3_ENABLE_ASN1` / `LIBE3_ENABLE_JSON` / `LIBE3_ENABLE_PROTOBUF` must be ON**; any combination can be built together and the encoder is chosen at runtime. `build_libe3` has a dedicated `--enable-protobuf` flag; JSON and SWIG use `--cmake-opt`. ## Architecture (big picture) @@ -58,7 +58,7 @@ The design has one façade over three pluggable seams, plus a real-time data pat **Dual-role façade.** `E3Agent` (`include/libe3/e3_agent.hpp`) is the single public class for **both** the RAN and dApp roles; the role is chosen by `E3Config.role` (RAN vs DAPP), not by a separate class. It is a Pimpl over the internal `E3Interface` (`src/core/e3_interface.cpp`), which coordinates the full protocol lifecycle (Setup → Subscription → Control/Indication → Release) and owns the worker threads and queues. `E3Config` (in `types.hpp`) is the one struct that selects role, transport, encoding, endpoints, and real-time tuning. **Three pluggable seams**, each behind an interface with a factory: -- **Transport** — `E3Connector` (`include/libe3/e3_connector.hpp`), implemented by the ZeroMQ and POSIX (TCP/SCTP/IPC) connectors in `src/connector/`, built by `connector_factory.cpp`. +- **Transport** — `E3Connector` (`include/libe3/e3_connector.hpp`), implemented by the ZeroMQ, POSIX and io_uring (TCP/SCTP/IPC) connectors in `src/connector/`, built by `connector_factory.cpp`. The POSIX and io_uring connectors share `socket_link.{hpp,cpp}` (endpoint, socket options, framing) so they stay the same wire; `uring_ring.cpp` is the only file that includes ``. See `docs/connectors/uring.md`. - **Encoding** — `E3Encoder` (`include/libe3/e3_encoder.hpp`), implemented by the encoders in `src/encoder/` (ASN.1 APER, JSON, and Protocol Buffers), built by `encoder_factory.cpp`. - **Service Model** — `ServiceModel` (`include/libe3/sm_interface.hpp`) is the RAN-side extension point for opaque, application-defined payload semantics. Its lifecycle is subscription-driven (`start()` on first subscriber to a RAN function, `stop()` on the last), managed via `SmRegistry` (`src/core/sm_registry.cpp`). diff --git a/README.md b/README.md index 71f3508e..d721d593 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ - **Vendor-Neutral API**: Clean E3Agent facade that hides implementation details - **Both Roles**: Single `E3Agent` class acts as either **RAN** (server, binds sockets) or **dApp** (client, connects to a remote RAN), switched by `E3Config.role` - **Multi-Peer dApps**: One dApp process can hold N `E3Agent` instances connecting to N different RAN agents (e.g. a DU-HIGH and a DU-LOW) -- **Multiple Transports**: Support for ZeroMQ and POSIX sockets (TCP, SCTP, Unix Domain) +- **Multiple Transports**: Support for ZeroMQ and POSIX sockets (TCP, SCTP, Unix Domain), the latter + also driven through io_uring on Linux (`E3LinkLayer::URING`, the same wire; see + [docs/connectors/uring.md](docs/connectors/uring.md)) - **Multiple Encodings**: ASN.1 APER (primary), JSON, and Protocol Buffers encoders - **Service Model Extensions**: Easy-to-implement SM interface for custom functionality - **Python Bindings**: Optional SWIG bindings (`LIBE3_ENABLE_SWIG=ON`) so the same C++ library can back Python dApp clients @@ -34,7 +36,7 @@ ▼ ▼ ┌───────────────────┐ ◄─── E3AP PDUs ───► ┌──────────────────────┐ │ E3Agent │◄──────────────────────────►│ E3Agent │ - │ │ ZMQ / POSIX │ │ + │ │ ZMQ / POSIX / io_uring │ │ │ register_sm(...) │ IPC · TCP · SCTP │ set_indication_ │ │ send_indication │ ASN.1 · JSON · Protobuf │ handler(fn) │ │ send_xapp_ctrl │ │ subscribe(rfid, …) │ @@ -63,7 +65,8 @@ ├────────────────────┤ ├──────────────────────┤ │ extension │ │ ZmqConnector │ │ Asn1Encoder │ │ point) │ │ PosixConnector │ │ JsonEncoder │ ├──────────────────┤ -└────────────────────┘ └──────────────────────┘ │ SimpleSM │ +│ UringConnector │ │ ProtobufEncoder │ │ SimpleSM │ +└────────────────────┘ └──────────────────────┘ │ │ │ custom SMs … │ └──────────────────┘ ``` @@ -80,6 +83,7 @@ - `protobuf-compiler` + `libprotobuf-dev` — `protoc` and the Protocol Buffers runtime (only when `LIBE3_ENABLE_PROTOBUF=ON`) - `libzmq3-dev` for ZMQ transport - `libsctp-dev` — SCTP headers for the POSIX/SCTP transport (only when `LIBE3_ENABLE_SCTP=ON`; **not installed by `./build_libe3 -I`**) +- `liburing-dev` (2.0 or later, Linux) for the io_uring link; without it the build goes on without the link ### Install Dependencies @@ -170,6 +174,7 @@ pkg-config --modversion libe3 | `LIBE3_ENABLE_TSAN` | OFF | Enable ThreadSanitizer | | `LIBE3_ENABLE_SWIG` | OFF | Build the SWIG-generated Python bindings (`_libe3py.so` + `libe3py.py`) | | `LIBE3_ENABLE_SCTP` | OFF | Enable the SCTP POSIX transport (needs `libsctp-dev`) | +| `LIBE3_ENABLE_URING` | ON | Build the io_uring link when it can be: on Linux with liburing found, otherwise the configure says so and goes on without it. `LIBE3_HAS_URING` (0 or 1) tells code and consumers which happened | > **Note on SCTP:** the transport is **off by default**. It is the only dependency that needs a > kernel module plus a distro `-dev` package for a single system header, and the deployed E3 links @@ -410,7 +415,7 @@ for local development (IPC) or production deployments. | `dapp_version` | string | dApp version advertised in `SetupRequest` | | `vendor` | string | Vendor name for `SetupRequest` | | `e3ap_version` | string | E3AP protocol version string | -| `link_layer` | E3LinkLayer | ZMQ or POSIX | +| `link_layer` | E3LinkLayer | ZMQ, POSIX or URING. URING is the POSIX link's wire driven through io_uring: every field below that names the POSIX link applies to it too, and its ring options come from `LIBE3_URING_*` environment variables ([docs/connectors/uring.md](docs/connectors/uring.md)) | | `transport_layer` | E3TransportLayer | SCTP, TCP, IPC | | `setup_endpoint` | string | Setup connection endpoint (default: ipc:///tmp/dapps/setup). The POSIX link serves everything on this one endpoint, with the IPC path /tmp/dapps/e3 while the field keeps its default; for TCP and SCTP `tcp://host:port` or `sctp://host:port` names the host and port | | `subscriber_endpoint` | string | Subscriber endpoint, ZeroMQ only (default: ipc:///tmp/dapps/dapp_socket) | diff --git a/docs/connectors/uring.md b/docs/connectors/uring.md new file mode 100644 index 00000000..452c16e1 --- /dev/null +++ b/docs/connectors/uring.md @@ -0,0 +1,516 @@ +# The io_uring link + +`E3LinkLayer::URING` drives the POSIX link's sockets through Linux io_uring. It puts the same +bytes on the same sockets as `E3LinkLayer::POSIX`: a RAN of one link serves a dApp of the other, +over IPC, TCP and SCTP alike. What changes is how a thread waits for those bytes and writes them, +and with that, the latency of the E3 hop and the CPU it costs. + +## What a user can do with it + +| Today, with `--link posix` | With `--link uring` | +|---|---| +| A received PDU costs two system calls (`poll`, then `recv`) and a thread wakeup | One `io_uring_enter` that re-arms the receive and waits; the wakeup stays | +| The receiving thread always sleeps between PDUs | It can poll the completion queue instead of sleeping (`LIBE3_URING_BUSY_POLL_US`), which takes the thread wakeup out of the hop: the wire hop fell from 7 to 2 µs on the x86 server measured below (from 20 to 3.5 to 6 µs on the Arm virtual machine), and the round trip by 12 to 48% (27 to 29% on Arm), for 1.1 to 1.6 times the CPU (2.4 times on Arm). With a kernel submission thread as well (`LIBE3_URING_SQPOLL`), a wire hop took 1.7 µs or less and a send 2.3 to 10 µs | +| A RAN sending to N dApps makes N `sendmsg` calls one after the other; a dApp that stopped reading holds the others back for up to the 500 ms send timeout | The N writes are submitted together; the others have their PDU at once, and only the call waits for the stalled one | +| Nothing to set up | Linux 5.11 or later, a libe3 built with liburing, and a process allowed to use io_uring (see [Where it runs](#where-it-runs)) | + +Nothing else changes: the same `E3Config` fields, endpoints, ports, `PosixOptions`, C API, +Python binding and threads. To use it, set `link_layer = E3LinkLayer::URING` (`--link uring` in +`example_simple_agent`, `example_simple_dapp` and `test_bench_full_loop_latency`, `2` in the C +API's `e3_config_t.link_layer`, `E3LinkLayer_URING` in Python). + +Which link to pick, from the measurements at the end of this page (two machines, an x86-64 server +seen through a 12-vCPU virtual machine and a 6-vCPU Arm virtual machine; interleaved rounds of 3000 +round trips per cell; IPC, TCP and SCTP; three encodings). The round trips below are ASN.1 medians on +the x86 server, with the Arm virtual machine's in brackets: + +- **Lowest latency, with cores to give it:** `uring` with `LIBE3_URING_SQPOLL=1` and + `LIBE3_URING_BUSY_POLL_US` above the gap between PDUs. It was the fastest on every transport and + encoding on both machines: 20.7 µs on IPC, 27.3 on TCP and 33.1 on SCTP, against POSIX's 50.4, 47.3 + and 48.6 (59, 42 and 32% less) [19.4, 19.6 and 34.3 against 110.5, 115.7 and 122.0], and 12.4 µs on + IPC with Protobuf against 29.9. It costs 1.4 to 2.0 times POSIX's CPU [2.6] and a spinning kernel + thread per connector. Its p99 was 27 to 51% below POSIX's on the x86 server (52 to 68 µs against 92 to + 111) but above it on the Arm virtual machine (218 to 305 against 165 to 180), where the spinning + threads shared six virtual CPUs with the rest. What the gain is made of differs too: on the x86 + server it is the connector's own rows (the wire hop and the send call, below); on the Arm virtual + machine it was mostly that no thread in the loop slept any more (the queuing rows fall from 19 µs to + 1.4 µs), and that holds only while PDUs come faster than libe3's queue spins (tens of microseconds). +- **Lower latency on any machine:** `uring` with `LIBE3_URING_BUSY_POLL_US` alone: 48, 27 and 12% below + POSIX on IPC, TCP and SCTP [27 to 29%], at 1.1 to 1.6 times the CPU [2.4], and a p99 15 to 49% lower on + the x86 server. +- **Otherwise POSIX; default io_uring is no better or worse than it, within the noise.** On the x86 + server io_uring without busy polling was 12% below POSIX on IPC, 5% above on TCP and 12% above on + SCTP, and the rounds of the two overlap on all three [0 to 4% above on Arm]. The wire hop is shorter on + IPC (4.8 against 7.2 µs) and the same on TCP, the send call costs 0.8 to 1.1 µs more, and the wakeups + that dominate both are the same. POSIX needs nothing from the kernel or the container. +- **`LIBE3_URING_SQPOLL` without busy polling is not a general choice:** the sending thread sleeps + until the kernel thread has done the write. On the x86 server it gave 35.6 µs on IPC (29% below + POSIX) but 48.3 on TCP (the same) and 56.2 on SCTP (16% above), at 1.5 to 1.9 times the CPU; on the + Arm virtual machine it was 31 to 45% above POSIX, at 3.5 to 3.8 times the CPU. +- ZeroMQ was the slowest: 1.8 to 2.1 times POSIX's round trip on the x86 server, twice on Arm. + +## Architecture + +``` + receive()/recv_setup_request() send()/send_to()/send_response() + the thread holding reader_mutex_ the thread holding tx_mutex_ + │ │ + ┌───────▼────────┐ ┌───────▼────────┐ + │ receive ring │ accept, one recv per │ send ring │ the writes of one call, + │ (UringRing) │ connection, a wake │ (UringRing) │ submitted together + └───────┬────────┘ eventfd read └───────┬────────┘ + │ │ + └────────────── sockets ──────────────────┘ + socket_link: endpoint, options, admission, framing + (shared with PosixE3Connector: one wire) +``` + +| File | What it holds | +|---|---| +| `src/connector/socket_link.{hpp,cpp}` | The endpoint parsing, the socket options, the admission rules and the framing, shared by the POSIX and the io_uring connector. Moved out of `posix_connector.cpp` without a change of behaviour; the POSIX tests are the proof | +| `src/connector/uring_ring.{hpp,cpp}` | One ring. The only file of libe3 that includes ``; completions come out as a plain `RingCompletion` | +| `src/connector/uring_connector.{hpp,cpp}` | `UringE3Connector`: the connections, the receive and send paths, the ring options | + +### Decisions, and what was not done + +**1. A connector of its own, on the POSIX wire.** The alternatives were a mode inside +`PosixE3Connector` (the benchmark could no longer put both in one build side by side, and that +file is already a thousand lines) and a new wire (it would split the peers into two camps that +cannot talk). Instead the bytes are the POSIX link's, which since #153 frames every transport the +same way in both directions, so mixed pairs work. The code that decides the bytes lives once, in +`socket_link`, so the two links cannot drift apart; the mixed pairs are tested (below). + +**2. The facade gains one enumerator.** `E3LinkLayer::URING = 2`, its string `uring`, the C API's +range, and the factory case. No new `E3Config` field and no new `E3Connector` method: the +`PosixOptions` apply as they are, because the sockets are the same. The ring options are tuning +for a deployment, not protocol, so they come from the environment (`LIBE3_URING_*`, read once +when the connector is made), the way latrec takes `LATREC_ENTRIES_LOG2`; they are not ABI and can +change without a version bump. A build without liburing keeps the enumerator and fails +`create_connector` with a message, exactly as a build without ZeroMQ does for `ZMQ`; there is no +silent fallback to POSIX, because a deployment that asked for io_uring and silently got something +else would be measured wrong. + +**3. The calling threads drive the rings; there is no I/O thread.** A dedicated I/O thread would +let `send()` return on enqueue, but it adds a thread hop to every PDU in both directions, which +costs a wakeup (about 20 µs on the machine below, a few microseconds on bare metal) and is +exactly what the link is meant to remove. So the thread that calls `receive()` waits on the +receive ring, and the thread that calls `send()` submits its writes and waits for their +completions. `send()` therefore returns when the kernel has the bytes, as with POSIX, and +`SEND_DONE` means the same on both links: the latrec rows compare one to one. + +**4. Two rings, one per direction.** A RAN's inbound thread waits on the receive ring for up to +half a second; the outbound thread must not wait behind it to submit a write. One ring each, +guarded by the mutex of its direction, keeps the two independent. With a kernel submission +thread the two rings share one (`IORING_SETUP_ATTACH_WQ`). + +**5. One receive armed per connection, re-armed with the wait.** The receive ring holds a receive +on every connection (and the accept, on a RAN). When `receive()` needs a PDU it re-arms what +completed and waits in the same `io_uring_enter`, so a PDU costs one system call where POSIX needs +two. A TCP receive may bring several PDUs; they are queued and handed out one per `receive()`, in +the order they came, with the close of a connection queued behind its last PDU so that +`take_closed_peers()` reports it only after that PDU was taken (the contract of `E3Connector`). +Multishot receives with provided buffers would also save the re-arm, but the wait stays one +system call per PDU when the thread sleeps, and they need kernel 6.0; they are deferred (below). + +**6. The kernel's default way of running completions, not `SINGLE_ISSUER`/`DEFER_TASKRUN`.** The +facade lets more than one thread read (a dApp's setup thread reads the setup response, then the +inbound thread reads the rest; a RAN's setup thread reads when nothing calls `receive()`) and more +than one thread send (setup and outbound). `IORING_SETUP_SINGLE_ISSUER` forbids that, and +`DEFER_TASKRUN` needs it. In the default mode the kernel finishes a completed receive on the +thread that armed it, and makes that thread enter the kernel with `TIF_NOTIFY_SIGNAL`, the +mechanism of a signal. Two consequences, both handled or written down: + +- A blocking call of that thread with a timeout returns `EINTR` when one of its receives + completes. The tests found it: a test thread that had read for the RAN then called `connect()` + (whose socket has a send timeout), and the connect failed. So a thread that reads only in + passing (the setup thread standing in for an idle `receive()`) takes back everything it armed + before it returns, and only the steady reader keeps receives armed. Within an `E3Agent` that is + the inbound thread, so **a Service Model's control handler, which runs on the RAN's inbound + thread, can see `EINTR` from a socket call that has `SO_RCVTIMEO` or `SO_SNDTIMEO`, or from + `epoll_wait`, while other dApps' receives are armed.** Calls without a timeout restart by + themselves. A dApp's handlers run while nothing of the dApp's is armed. With a kernel + submission thread none of this happens: that thread arms and runs everything. +- Per-thread rings with `SINGLE_ISSUER | DEFER_TASKRUN` would remove both the interruption and the + wakeups it costs, at the price of a ring per thread, the registered files kept per ring, and a + hand-over between the setup and the inbound thread. Deferred, with its cost, below. + +**7. Busy polling and the kernel submission thread are opt-in.** Both trade CPU for latency, and +neither is free to switch on in a shared machine: + +- `LIBE3_URING_BUSY_POLL_US=N`: a thread waiting for a completion reads the completion queue for up + to N µs before it sleeps. Without a submission thread the kernel still needs the waiting thread + to finish the receive, and interrupts it to do so (an inter-processor interrupt, a few hundred + nanoseconds) instead of waking it from sleep (the scheduler, a C-state exit, and in a VM a halted + virtual CPU: about 20 µs below). It keeps one core busy per receiving thread for the gap between + PDUs. A receiving thread that busy-polls must not share its core with the sending thread: + `io_thread_affinity` pins both I/O threads of an agent to one core, so do not combine the two. +- `LIBE3_URING_SQPOLL=1`: a kernel thread takes the submissions and finishes the completions, so + neither direction makes a system call, and no user thread is interrupted. It spins for a second + after the last submission (`sq_thread_idle`), on the core of `LIBE3_URING_SQPOLL_CPU` if set. + With busy polling too it gave a wire hop under a microsecond and the fastest round trip measured; + without busy polling it is the slowest variant, because the sending thread then sleeps until the + kernel thread has written. It needs cores of its own: on a busier machine its spinning threads + competed with libe3's others and it fell behind busy polling alone. + +**8. Registered files on, registered buffers off, by default; neither measurably matters.** Each +connection takes a slot in both rings' file tables (`IORING_REGISTER_FILES`, sparse), so an operation skips the descriptor +lookup and its reference count; the first 256 connections get a slot and later ones use the plain +descriptor. Registered buffers (`READ_FIXED` into a buffer registered per connection) are offered +for the measurement but left off: a socket receive copies the bytes either way, so all they save is +the user-pointer checks, and they pin 64 to 128 KiB per connection against `RLIMIT_MEMLOCK`. SCTP +never uses them, as it needs `recvmsg` for the payload protocol id. Each falls back to the plain +way when the kernel refuses it, and `UringE3Connector::settings()` says what is in effect. Measured, +neither registered files nor registered buffers moved the round trip beyond the spread of the +rounds: the descriptor lookup they save is tens of nanoseconds in a hop of microseconds. Files stay +on because they cost nothing on the data path. + +**9. A broadcast is one batch.** `send_to(0, …)` prepares one write per connection and submits +them together. Writes that complete at once (the usual case) cost one system call for all of them. +A write still pending after `SEND_TIMEOUT_MS` (500 ms, as POSIX) is cancelled, its connection is +shut, and the reader then reports it closed, which is what POSIX does after `SO_SNDTIMEO`. A +short TCP write is completed by submitting the rest. The call returns only when the kernel holds no +reference to the caller's buffer. + +**10. Shutdown wakes the reader through an eventfd.** `shutdown()` is the one call made from +another thread while a reader waits. It shuts the sockets, as POSIX does, and also writes an +eventfd whose read is armed on the receive ring, so a reader that waits on nothing else (a RAN with +no dApp) returns at once. `dispose()` takes back everything still armed before it frees the +buffers, then tears the rings down; it can be called twice. + +**11. SCTP bounds its retransmission timeout, on both socket links.** Repeating the tests found a +send to a dApp over SCTP that stalled past the 500 ms send timeout, so the RAN dropped the dApp. It +was not io_uring: a burst of 2000 PDUs of up to 64 KiB from the RAN dropped the dApp in 8 of 100 +runs on POSIX and 7 of 100 on io_uring, and never over TCP. A burst closes the receiver's window, +and the sender then waits on SCTP's retransmission timer, whose smallest value in Linux is 1 s +(`net.sctp.rto_min`). With the minimum at 20 ms the same runs never dropped a dApp and the +slowest send took 25 ms. So the shared socket setup gives every association a timeout of 120 ms at +first and 20 to 250 ms after (`SCTP_RTOINFO`, the per-socket form of the sysctl, the same idea as +the 120 and 500 ms that telecom SCTP stacks configure); 250 ms is half the send timeout, so one +backed-off wait cannot reach it. With it, the same 100 runs at the kernel's default sysctl dropped no +dApp on either link (slowest send 23 to 25 ms). E3 runs between neighbours (the same host, or one switch apart), +where 20 ms is still far above any round trip. It changes the POSIX link's SCTP timers as well, +which keeps the two links the same; `test_socket_link` reads the values back from the kernel. + + +## Where it runs + +| Requirement | Why | What happens otherwise | +|---|---|---| +| Linux 5.11 or later | `IORING_FEAT_EXT_ARG` (waits with a timeout without a ring entry) and `IORING_FEAT_FAST_POLL` | `create_connector` returns null and logs "Linux 5.11 or later is needed" | +| liburing 2.0 or later at build time (`liburing-dev`, `liburing-devel`) | the ring API | the link is not built; `--link uring` fails with "built without it" | +| io_uring allowed for the process | Docker's default seccomp profile refuses `io_uring_setup`, and `kernel.io_uring_disabled` set to 1 (allowed for one group) or 2 (nobody) refuses it on the host; Red Hat Enterprise Linux 9 and so OpenShift's nodes ship 2, which is why the x86 measurements below were taken in a virtual machine on such a cluster | `create_connector` returns null and logs which of the two to change: `--security-opt seccomp=unconfined` or a profile that allows `io_uring_setup`, `io_uring_enter` and `io_uring_register`; or the sysctl | +| `RLIMIT_MEMLOCK` | only for registered buffers | the buffer falls back to a plain one, with a warning | + +The tests that need io_uring say `URING NOT RUN` and exit 77 (CTest's "skipped") where it is +refused, unless `LIBE3_REQUIRE_URING` is set: then they fail, so that CI cannot pass by skipping. + +## Ring options + +Read from the environment when the connector is made (`E3Interface::init()`, or each restart of a +dApp). A value out of range is ignored with a warning. + +| Variable | Default | Range | What it does | +|---|---|---|---| +| `LIBE3_URING_DEPTH` | 64 | 4 to 4096 | Submission queue entries of each ring; the completion queue has four times as many. A broadcast to more connections than this is submitted in pieces | +| `LIBE3_URING_FIXED_FILES` | 1 | 0, 1 | Registered files (decision 8) | +| `LIBE3_URING_FIXED_BUFFERS` | 0 | 0, 1 | Registered receive buffers on TCP and IPC (decision 8) | +| `LIBE3_URING_BUSY_POLL_US` | 0 | 0 to 10000000 | Poll the completion queue this long before sleeping (decision 7) | +| `LIBE3_URING_SQPOLL` | 0 | 0, 1 | A kernel submission thread per connector (decision 7) | +| `LIBE3_URING_SQPOLL_CPU` | -1 | -1 to 4095 | The CPU of that thread; -1 lets the scheduler place it | + +## For a real-time deployment + +- Give each busy-polling receiving thread a core of its own, isolated from the scheduler + (`isolcpus`, `nohz_full`, `rcu_nocbs`, as `libe3-paper/scripts/run_baseline.sh` does), and set the + budget above the gap between PDUs (an indication per slot at 30 kHz is 500 µs). A budget shorter + than the gap still sleeps, only later. +- Do not pin a busy-polling agent with `io_thread_affinity`: it pins the inbound and the outbound + thread to the same core, and the spinning one starves the other. +- With `LIBE3_URING_SQPOLL`, pin the kernel thread (`LIBE3_URING_SQPOLL_CPU`) to an isolated core + as well; one per connector, so two for a RAN and a dApp in one process. +- In a container: allow io_uring in the seccomp profile. In Kubernetes and OpenShift that is a + `seccompProfile` of type `Localhost` (or `Unconfined`), and on a node whose kernel ships + `kernel.io_uring_disabled=2`, a node-level sysctl change. +- An SM control handler that makes socket calls with a timeout on the RAN's inbound thread should + retry on `EINTR` (decision 6), or the deployment should use the kernel submission thread. + +## Tests + +Written before the code, in the order of the issue's plan, and run on Linux 6.8 on two machines (an +x86-64 server's virtual machine and an Arm virtual machine) and, early on, on 7.0 in a Docker +container with seccomp unconfined: the whole suite with io_uring and SCTP required (81 of 81 on +x86-64), `build_libe3` from a clean tree (62 of 62), the io_uring and link-matrix tests fifteen +times over, the connector tests under AddressSanitizer with UndefinedBehaviorSanitizer (no report) +and under ThreadSanitizer (every report is inside libzmq's own threads), the example RAN and dApp as +two processes on every pair of links and transport (14 of 14), and a real gNB (below). + +- **The wire, against itself and against POSIX.** The POSIX connector and pair tests run again, + unchanged, with `--link uring`, and with a RAN of one link and a dApp of the other + (`tests/link_under_test.hpp`; CTest names `*_uring`, `*_uring_ran_posix_dapp`, + `*_posix_ran_uring_dapp`): `test_role_aware_connector_posix` (framing on every transport, the + old layout refused, `send_to` to one peer, the peer list, the cap, IPv6), + `test_dapp_bound_to_connection_posix`, `test_dapp_link_closed_posix`, and the integration tests + `test_role_pair_posix` and `test_role_pair_sctp` (payload protocol id and streams read off the + association by a proxy). These extra runs set `LATREC_ENTRIES_LOG2=12`: in a latrec build every + agent thread leaves a pre-faulted ring in the build tree, 8 MiB by default, and four runs of each + test would otherwise fill a small disk when the suite is repeated. +- **Only io_uring** (`test_uring_connector`): each ring variant (plain, registered files, registered + buffers, submission thread, busy polling, all together) carries a setup and both directions on + IPC, TCP and SCTP and is the variant in effect; a broadcast to more connections than ring entries; + a stalled peer does not delay the others, which have their PDU long before the send gives up on + it; a reset peer is dropped alone; 2000 PDUs of every size up to 65536 octets, both ways, on + every transport; the size limit; the receive timeout; `shutdown()` waking a waiting `receive()`. +- **The agent over it**: `test_dapp_ran_loss`, `test_dapp_reconnect`, + `test_dapp_refused_setup_reconnect`, `test_multi_peer_dispatch` (including the slow-peer case) and + `test_latrec_stages` (every PDU sent is stamped `SEND_DONE` after `ENCODE_E3AP_DONE`) each gained + io_uring cases. +- **The build**: `test_link_availability` runs in every configuration and checks that the link is + there exactly when built, that a build without it refuses `--link uring`, and that link layer 2 + of the C API is the io_uring link. +- **The framing** shared by both links, as pure functions (`test_socket_link`): split frames, two + frames in one read, a length of zero or above the limit, the endpoint parsing; and the SCTP + retransmission timeout of decision 11, read back from the kernel. +- **Ports.** The tests take their ports from `free_ports()` (`tests/link_under_test.hpp`): below the + range the kernel uses for the source ports of outgoing connections, bound once to see that they are + free, and claimed with a lock on a file that the process holds until it exits, so that no two tests + take the same port however they order asking and listening. Ports from inside the source-port range + collided with a source port still in TIME_WAIT (two runs in thirty of `test_uring_connector`), and + ports derived from the pid with a different base in each test overlapped each other (eight runs in + twelve of the connector tests at `-j3`). +- **A real gNB** (not in CTest: it needs an OAI build, about 10 GB): the OAI gNB with its E3 agent + (`--E3Configuration.link uring`) and one nrUE on the RF simulator, and the dApp library's spectrum + dApp over the other end, on IPC, TCP and SCTP with the RAN and the dApp each on POSIX or io_uring: 12 + of 12 pairs run, 16,000 to 18,600 sensing slots handled in 15 s and 99.7 to 99.9% of them with their + sensing ranges, no assertion in the gNB. Two things were found there that are not about io_uring: + an IPC endpoint is created with the permissions of the RAN's user, so a dApp of another user cannot + connect to it (the same on POSIX), and the dApp library's spectrum dApp asks the L1-KPM function for a + control identifier that it does not have, which libe3 now refuses (#96); the run used an empty list. +- **The comparison tool** (`test_bench_matrix`): the numbers it reads from the benchmark's table, and + its verdict, on canned output. + +Issue #140 proposed a portable layer under the connector (a completion loop fed by a fake ring) so +that the completion handling could run on macOS. It was not built: since #153 the library targets +Linux only, the framing is tested as pure functions instead, and the completion handling (a reset +peer, a cancelled write, short writes, a full queue) with real sockets, where the kernel's own +behaviour is what is being tested. + +CI: the unit and integration jobs set `LIBE3_REQUIRE_URING`; the end-to-end matrix gained +`uring` (and SCTP for POSIX and io_uring); a build with `-DLIBE3_ENABLE_URING=OFF` runs the suite; +the full-loop stage runs the comparison below on its runner and puts it in the CI report. + +## Measurements + +**How.** `tools/bench_matrix.py` runs `test_bench_full_loop_latency` (a RAN and a dApp in one +process, the Simple Service Model in ping-pong: one indication in flight, the next emitted when the +control comes back) for each link variant and transport, in rounds that interleave the cells so a +drift of the machine hits them all alike, and reads back the 18-phase latrec table of each run. The +round trip is the primary number; the send call (A6, A15), the wire hop (`SEND_DONE` to `RECV`) and +the CPU of the process per round trip, both ends together, explain it. `send()` returns when the +kernel has the bytes on both socket links, so their rows compare one to one. The CI full-loop stage +runs the same comparison (ZeroMQ, POSIX, io_uring, and io_uring with busy polling, with and without +the submission thread, on IPC, TCP and SCTP, three rounds) and puts the table in the CI report. + +A wire hop can come out below zero. `SEND_DONE` is stamped after `send()` has returned, and a receiver +that polls can have the PDU, and stamp `RECV`, before that: SCTP over loopback delivers inside the +sender's system call, and io_uring completes a send on a kernel thread. The benchmark pairs the dApp's +inbound records with the RAN's outbound ones by position, and checks each pair for the right order, +allowing the stamps 5 µs of skew (a pair that is off by one round trip is off by 12 µs or more). Without +that allowance, on the x86 server, SCTP runs with busy polling were refused as misaligned in about one +run in ten (12 of 110; nothing was lost, the counts of PDUs out and in agreed); the tables below show the +hop as it is, below zero where it is. The Arm tables were taken with the stricter check, so their SCTP wire +figures may sit a little high. + +**Where.** Two machines, to see what depends on the machine. The tables in the first subsection are +the reference; the second is the earlier campaign. + +### On the x86 server + +An Intel Xeon Gold 6240R (Cascade Lake, 2.4 GHz) server of a Kubernetes cluster, seen through a KubeVirt +virtual machine of 12 virtual CPUs and 24 GB: Ubuntu 24.04, Linux 6.8, liburing 2.5, Release build, +latrec on, nothing pinned or isolated, other workloads on the host (it was lightly loaded). It had to be +a virtual machine: the host kernels of that cluster set `kernel.io_uring_disabled=2` (the RHEL default), +which hides io_uring from every container on them. Waking a sleeping thread costs a few microseconds +here, less than on the Arm virtual machine below, so the wakeups weigh less. Not bare metal and not +isolated cores: the p99 and the spread of the rounds show it. + +**Round trip, ASN.1, nine rounds of 3000 round trips each** (median over the rounds of each round's +statistic, the range of the rounds in brackets, microseconds; CPU is the process, both ends, per round +trip; `busy` is `LIBE3_URING_BUSY_POLL_US=1000`, `sqpoll` is `LIBE3_URING_SQPOLL=1`, `plain` is +registered files off, `fixedbuf` registered buffers on): + +| Transport | Link | Round trip p50 | p99 | Wire RAN to dApp p50 | Wire dApp to RAN p50 | Send indication p50 | Send control p50 | CPU | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| IPC | zmq | 88.8 [65.1 to 92.8] | 158.6 | 23.8 | 23.6 | 3.6 | 3.6 | 186 | +| IPC | posix | 50.4 [41.4 to 53.7] | 105.7 | 7.2 | 7.8 | 4.0 | 3.7 | 138 | +| IPC | uring | 44.1 [30.2 to 49.8] | 94.9 | 4.8 | 5.1 | 4.8 | 4.7 | 128 | +| IPC | uring, plain | 41.3 [31.6 to 50.5] | 80.4 | 4.8 | 4.9 | 4.7 | 4.6 | 121 | +| IPC | uring, fixedbuf | 40.5 [38.5 to 50.7] | 80.5 | 4.9 | 4.9 | 4.8 | 4.5 | 122 | +| IPC | uring, busy | 26.2 [23.2 to 29.2] | 53.8 | 2.0 | 2.2 | 4.0 | 4.0 | 149 | +| IPC | uring, sqpoll | 35.6 [27.7 to 41.2] | 83.1 | 1.8 | 1.8 | 7.3 | 7.7 | 201 | +| IPC | **uring, sqpoll + busy** | **20.7** [18.2 to 22.6] | 51.6 | 1.6 | 1.7 | 2.4 | 2.3 | 194 | +| TCP | zmq | 98.9 [73.4 to 102.2] | 189.6 | 30.1 | 27.9 | 3.8 | 3.7 | 204 | +| TCP | posix | 47.3 [41.6 to 53.5] | 92.0 | 3.3 | 3.7 | 8.0 | 7.7 | 129 | +| TCP | uring | 49.6 [42.1 to 56.0] | 99.9 | 3.4 | 3.4 | 9.1 | 9.0 | 132 | +| TCP | uring, plain | 52.5 [44.1 to 56.6] | 114.5 | 3.4 | 3.6 | 9.2 | 9.1 | 142 | +| TCP | uring, fixedbuf | 53.1 [43.6 to 58.4] | 113.7 | 4.0 | 3.6 | 9.8 | 9.3 | 144 | +| TCP | uring, busy | 34.5 [26.4 to 36.6] | 66.9 | 0.8 | 1.0 | 8.1 | 7.8 | 183 | +| TCP | uring, sqpoll | 48.3 [34.8 to 54.2] | 96.6 | 1.0 | 1.1 | 11.1 | 11.0 | 233 | +| TCP | **uring, sqpoll + busy** | **27.3** [25.8 to 31.1] | 67.6 | 0.8 | 1.0 | 5.8 | 5.4 | 235 | +| SCTP | posix | 48.6 [33.8 to 55.1] | 110.7 | 0.2 | 0.7 | 13.2 | 12.7 | 139 | +| SCTP | uring | 54.2 [35.8 to 57.4] | 109.2 | 0.1 | 0.2 | 14.1 | 14.2 | 148 | +| SCTP | uring, plain | 57.0 [41.3 to 61.8] | 116.2 | 0.1 | 0.2 | 14.4 | 14.1 | 154 | +| SCTP | uring, fixedbuf | 52.2 [48.8 to 56.5] | 121.5 | 0.0 | 0.2 | 14.0 | 14.2 | 148 | +| SCTP | uring, busy | 43.0 [37.1 to 46.7] | 93.8 | -1.5 | -1.2 | 13.5 | 13.3 | 217 | +| SCTP | uring, sqpoll | 56.2 [40.0 to 61.8] | 110.9 | -0.6 | -0.4 | 15.6 | 15.2 | 261 | +| SCTP | **uring, sqpoll + busy** | **33.1** [29.1 to 40.6] | 66.5 | -0.8 | -0.7 | 9.9 | 9.7 | 271 | + +**The same with JSON and Protobuf**, seven rounds of 3000, round trip p50 (the order of the links does +not change with the encoding): + +| Transport | Encoding | posix | uring | uring, busy | uring, sqpoll + busy | +|---|---|---:|---:|---:|---:| +| IPC | JSON | 62.2 | 57.1 | 47.3 | **39.7** | +| IPC | Protobuf | 29.9 | 25.6 | 17.1 | **12.4** | +| TCP | JSON | 64.5 | 67.7 | 56.6 | **46.5** | +| TCP | Protobuf | 29.5 | 32.2 | 23.7 | **17.1** | +| SCTP | JSON | 67.3 | 70.5 | 64.0 | **50.2** | +| SCTP | Protobuf | 32.1 | 33.0 | 27.7 | **23.0** | + +**Fastest: io_uring with the kernel submission thread and busy polling**, on every transport and +encoding, 59, 42 and 32% below POSIX with ASN.1. Next comes io_uring with busy polling alone. What each +number is made of, from the 18-phase table of one run each (IPC, ASN.1, 3000 round trips, p50 in +microseconds): + +| Phase | posix | uring | uring, busy | uring, sqpoll + busy | +|---|---:|---:|---:|---:| +| Queuing, indication (4) | 0.89 | 0.66 | 0.55 | 0.55 | +| Send, indication (5) | 4.26 | 5.64 | 4.09 | 2.47 | +| Wire, RAN to dApp (6) | 7.02 | 4.49 | 2.03 | 1.59 | +| Decode, indication (7) | 1.54 | 1.53 | 1.46 | 1.23 | +| Queuing, control (13) | 0.69 | 0.72 | 0.58 | 0.54 | +| Send, control (14) | 4.56 | 4.46 | 4.07 | 2.38 | +| Wire, dApp to RAN (15) | 7.97 | 5.01 | 2.19 | 1.67 | +| Decode, control (16) | 1.64 | 1.51 | 1.55 | 1.29 | +| **Round trip** | **43.95** | **37.32** | **25.84** | **21.99** | +| CPU per round trip, both ends | 138.8 | 123.0 | 144.2 | 209.5 | +| Voluntary context switches per round trip | 3.50 | 3.41 | 1.09 | 1.11 | + +- **The wire hop and the send call are where the connector acts.** io_uring's re-armed receive makes + the hop 2.5 µs shorter than POSIX's `poll` and `recv` and its send call 1.4 µs longer (the submission + and the wait for the completion). Busy polling takes the receiver's wakeup out of the hop (7.0 to 2.0 + µs, and 3.5 voluntary context switches per round trip fall to 1.1). The kernel submission thread makes + the send a write to shared memory (4.1 to 2.5 µs) and the hop 1.6 µs. +- **No second effect from the queues.** The queuing rows are 0.5 to 0.9 µs at the median in every + configuration here, because a thread wake costs a few microseconds, not twenty, and libe3's queue spins + before it sleeps. On the Arm virtual machine below the same rows cost 19 µs and fell to 1.4 µs in the + fastest configuration, which was most of its gain; here it is not. +- **SCTP's wire hop is about zero** and its send call is longer (13 µs against 8 µs for TCP and 4 µs for + IPC): over loopback the kernel delivers the packet inside the sender's system call. For SCTP the send + and the wire hop are to be read together, and a hop below zero is the receiver having the PDU before the + sender has stamped its return. +- **What does not matter.** Registered files and registered buffers made no difference beyond the spread + of the rounds (the `plain` and `fixedbuf` rows). +- **What it costs.** Busy polling 1.1 to 1.6 times POSIX's CPU, the submission thread with busy polling + 1.4 to 2.0 times, the submission thread alone 1.5 to 1.9 times. The ranges of the rounds are wide + (a tenth to a third of the median), as expected of a virtual machine on a shared host; where two links' + ranges overlap the tool says the order is within the noise rather than naming a winner. + +### On the Arm virtual machine + +A Linux virtual machine on an Apple Silicon Mac (Lima, Apple's Virtualization framework): +Ubuntu 24.04, Linux 6.8, 6 virtual CPUs, 4 GB, liburing 2.5, nothing else running, nothing pinned +or isolated. Release build, latrec on. In a virtual machine waking a sleeping thread is slow (an +idle virtual CPU is halted and must be woken through the hypervisor), about 20 µs here against a +few on bare metal, so every hop that waits for a wakeup is inflated, and whatever avoids sleeping +gains more here than it would on bare metal. The absolute numbers are an upper bound; the order of +the links, and the size of each connector row, are what to read. + +**Round trip, ASN.1, five rounds of 3000 round trips each** (median over the rounds of each round's +statistic, the range of the rounds in brackets, microseconds; CPU is the process, both ends, per +round trip; `busy` is `LIBE3_URING_BUSY_POLL_US=1000`, `sqpoll` is `LIBE3_URING_SQPOLL=1`, `plain` is +registered files off, `fixedbuf` registered buffers on): + +| Transport | Link | Round trip p50 | p99 | Wire RAN to dApp p50 | Wire dApp to RAN p50 | Send indication p50 | Send control p50 | CPU | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| IPC | zmq | 208.2 [207.2-232.1] | 297.2 | 69.0 | 69.4 | 6.0 | 6.1 | 194 | +| IPC | posix | 110.5 [110.1-129.8] | 164.8 | 21.2 | 23.0 | 6.6 | 6.1 | 136 | +| IPC | uring | 111.9 [111.5-137.5] | 181.1 | 21.2 | 22.1 | 7.5 | 7.8 | 140 | +| IPC | uring, plain | 113.3 [110.5-154.5] | 196.9 | 21.9 | 22.1 | 7.9 | 8.1 | 140 | +| IPC | uring, fixedbuf | 113.4 [111.0-133.6] | 179.7 | 21.2 | 22.0 | 7.5 | 7.7 | 140 | +| IPC | uring, busy | 78.4 [78.2-79.1] | 172.6 | 6.1 | 5.9 | 7.1 | 7.0 | 325 | +| IPC | uring, sqpoll | 145.0 [144.0-158.2] | 264.5 | 17.8 | 16.8 | 22.7 | 21.2 | 505 | +| IPC | **uring, sqpoll + busy** | **19.4** [15.3-19.5] | 265.1 | 0.6 | 0.8 | 2.5 | 2.1 | 354 | +| TCP | zmq | 221.8 [220.9-226.7] | 325.8 | 74.2 | 75.4 | 6.1 | 6.3 | 209 | +| TCP | posix | 115.7 [113.5-118.3] | 174.6 | 20.1 | 20.2 | 10.5 | 10.2 | 141 | +| TCP | uring | 119.0 [118.5-119.3] | 179.7 | 19.5 | 20.8 | 11.5 | 11.9 | 145 | +| TCP | uring, plain | 118.7 [117.9-120.5] | 184.7 | 20.0 | 20.2 | 11.7 | 11.8 | 146 | +| TCP | uring, fixedbuf | 120.4 [119.5-122.0] | 185.3 | 19.8 | 21.3 | 11.7 | 12.0 | 148 | +| TCP | uring, busy | 84.1 [84.1-85.0] | 154.1 | 5.1 | 4.8 | 11.0 | 11.0 | 338 | +| TCP | uring, sqpoll | 165.7 [165.4-167.5] | 256.6 | 16.2 | 16.7 | 29.8 | 29.4 | 531 | +| TCP | **uring, sqpoll + busy** | **19.6** [18.0-21.2] | 305.5 | 0.2 | 0.3 | 3.8 | 3.4 | 378 | +| SCTP | posix | 122.0 [120.5-123.1] | 180.0 | 19.2 | 19.4 | 14.1 | 13.8 | 150 | +| SCTP | uring | 126.9 [126.0-128.1] | 192.7 | 20.1 | 18.8 | 15.6 | 15.3 | 156 | +| SCTP | uring, plain | 126.4 [126.2-127.2] | 186.3 | 20.0 | 18.4 | 15.6 | 15.5 | 156 | +| SCTP | uring, fixedbuf | 127.1 [125.8-130.3] | 198.7 | 19.9 | 18.2 | 15.7 | 15.4 | 157 | +| SCTP | uring, busy | 88.4 [88.2-88.8] | 149.8 | 3.5 | 3.3 | 13.9 | 14.3 | 355 | +| SCTP | uring, sqpoll | 176.9 [174.5-179.0] | 265.8 | 9.4 | 12.5 | 31.2 | 32.6 | 525 | +| SCTP | **uring, sqpoll + busy** | **34.3** [32.0-34.7] | 218.6 | 0.7 | 1.0 | 8.2 | 8.0 | 393 | + +**The same with JSON and Protobuf**, three rounds of 2000, round trip p50 (the order of the links +does not change with the encoding): + +| Transport | Encoding | posix | uring | uring, busy | uring, sqpoll + busy | +|---|---|---:|---:|---:|---:| +| IPC | JSON | 123.4 | 124.2 | 90.5 | **37.6** | +| IPC | Protobuf | 106.6 | 106.3 | 75.1 | **14.2** | +| TCP | JSON | 127.0 | 133.3 | 94.7 | **43.8** | +| TCP | Protobuf | 112.6 | 116.5 | 82.0 | **13.7** | + +**Fastest: io_uring with the kernel submission thread and busy polling**, on every transport and +encoding, its slowest round faster than any other link's fastest. Next comes io_uring with busy +polling alone, 27 to 29% below POSIX. What each number is made of: + +- **The connector's own rows.** Busy polling takes the receiver's wakeup out of the wire hop (about + 20 µs to 3.5 to 6 µs). The kernel submission thread makes the send a write to shared memory and the + wire hop a read of it (a wire hop of 0.2 to 1.0 µs; a send of 2 to 8 µs against 6 to 14 for POSIX). + Without busy polling the submission thread only adds a wakeup to every send (21 to 33 µs). +- **Not the connector's.** The two queuing hops in front of the outbound threads (rows 4 and 13 of the + benchmark's table) cost about 19 µs each in every configuration but one: with the submission thread + and busy polling, the next PDU reaches the outbound thread while it is still spinning in libe3's + queue (40 spins, then 100 yields) and they fall to 1.4 µs, because no thread in the loop sleeps any + more. That is most of the drop from 78 to 19 µs, and it holds only while PDUs come faster than the + queue spins. Per phase, IPC, one run of 3000: POSIX 18.7 / 6.5 / 21.5 µs for queuing / send / wire + of the indication, busy polling 19.9 / 7.2 / 6.2, submission thread and busy polling 1.3 / 2.4 / 0.6. +- **What does not matter.** Registered files and registered buffers made no difference beyond the + spread of the rounds (the `plain` and `fixedbuf` rows); the send call of default io_uring costs about + 1 µs more than POSIX's `sendmsg` in the loop, 0.2 to 0.5 µs of it measured in isolation against the + same plain receiver, and that is why io_uring without busy polling is 0 to 4% behind POSIX. +- **What it costs.** Busy polling 2.4 times POSIX's CPU, the submission thread and busy polling 2.6 + times, the submission thread alone 3.5 to 3.8 times. The p99 of the submission thread with busy + polling is the worst of the fast variants: the spinning threads and the two kernel threads share six + virtual CPUs with libe3's other threads. + +**Run to run.** A first, preliminary run in a Docker Desktop virtual machine (10 virtual CPUs, more +threads around, one round of 500) agreed with the Arm virtual machine on everything but the order of +the two fastest variants: there the submission thread with busy polling took 86 µs and busy polling +alone 77 µs, as the spinning threads competed with the rest and the loop did not stay out of sleep. +The x86 server differs from both on three counts: the link with the submission thread and busy polling +was first there too, but with a lower p99 than POSIX's, not a higher one; the queuing rows play no part; +and default io_uring was not slower than POSIX beyond the noise. So the first place needs cores of its +own and a quiet machine; the second needs neither. + +**Not measured:** bare metal with isolated cores (the x86 server was a virtual machine too); two hosts over a network; more than one dApp per RAN +(the batching of a broadcast); periodic rather than ping-pong traffic. + +## Deferred, with their cost + +| Item | Cost to the user | Status | +|---|---|---| +| Per-thread rings with `SINGLE_ISSUER` and `DEFER_TASKRUN` (decision 6) | A blocking call with a timeout on the RAN's inbound thread can return `EINTR` while receives are armed; each completion interrupts the waiting thread | Not scheduled | +| A dedicated I/O thread and an asynchronous `send()` | `send()` waits for the kernel as with POSIX | Not planned: it adds the thread hop the link removes | +| Multishot receive, provided-buffer rings, `SEND_ZC` (kernel 6.0 and later) | A receive is re-armed for each PDU; no zero-copy send for large PDUs (which would not help on loopback or IPC anyway) | Not scheduled; the probe is in place | +| NAPI busy polling (`IORING_REGISTER_NAPI`) | Busy polling stops at the socket; a real NIC still interrupts | Not scheduled; meaningless on loopback and IPC | +| Connector-level latrec stamps (submission, completion) | The send and wire rows cannot be split into submission, kernel and completion time | Unowned | +| The io_uring link in the Python wheels | `pip install libe3py` gets a library without the link (manylinux has no recent liburing); a Python dApp needs libe3 built from source | Unowned | +| A run on bare metal with isolated cores, and across two hosts | The numbers below come from a virtual machine and loopback | Unowned | +| `uring` in the dApp library and in OAI's E3 agent configuration | Their configuration parsers know `zmq` and `posix` only, so their users cannot pick the link yet; nothing breaks | A few lines in each; the dApp library's was run (its Python dApp over io_uring, against both links); twin pull requests not opened | diff --git a/docs/control-loops/path-a-e3-loop.md b/docs/control-loops/path-a-e3-loop.md index 69cd79d4..7c594239 100644 --- a/docs/control-loops/path-a-e3-loop.md +++ b/docs/control-loops/path-a-e3-loop.md @@ -111,6 +111,19 @@ happens later on ZeroMQ's internal I/O thread. The `SEND_DONE` to `RECV` segment therefore contains that hop for ZeroMQ but not for the POSIX connectors, and any comparison between the two shall say so. +The io_uring connector returns from `send()` once the kernel has the bytes, as +the POSIX one does (the calling thread submits the write and waits for its +completion), so A6, A15 and the wire hops compare row by row between those two +links. With busy polling (`LIBE3_URING_BUSY_POLL_US`) the receiving thread does +not sleep, which is what shortens the wire hop; with a kernel submission thread +(`LIBE3_URING_SQPOLL`) the write is issued by that thread. See +[docs/connectors/uring.md](../connectors/uring.md). + +`SEND_DONE` is stamped after `send()` has returned, and a receiver that polls can have the PDU, and +stamp `RECV`, before that: SCTP over loopback delivers inside the sender's system call, and io_uring +completes a send on a kernel thread. Such a hop reads a little below zero (a microsecond or two); +the benchmark reports it as it is and does not take it for a misaligned pairing. + ## Return leg: dApp to RAN (`leg=ctrl_down` for control, `report_up` for reports) | # | Box | Owner | Segment | diff --git a/docs/e3ap/04-encoding.md b/docs/e3ap/04-encoding.md index 00f57ae2..7278261f 100644 --- a/docs/e3ap/04-encoding.md +++ b/docs/e3ap/04-encoding.md @@ -276,6 +276,11 @@ The POSIX link layer uses one connection for each dApp, to one endpoint. The set channel, the inbound channel and the outbound channel all travel on it. The E3 Agent listens, and the dApp connects. +The present document defines what travels on the sockets, not how an implementation +waits for them or writes to them. An implementation may drive them with ordinary system +calls or through an asynchronous interface of the operating system, such as io_uring on +Linux; to a peer the two are the same link. + The first E3-PDU of a connection is the SETUP REQUEST message of the dApp. Every later E3-PDU of the connection belongs to the dApp that sent it. A dApp that sends its SETUP REQUEST again, after a refusal, uses a new connection. The E3 Agent ties the diff --git a/docs/e3ap/O-RAN.XXX.TS.E3AP-R000-v00.01.docx b/docs/e3ap/O-RAN.XXX.TS.E3AP-R000-v00.01.docx index 8a0128bbb6d1aba1338f725e37f3cd6c81322ade..1e6aec1aaa01d263bb639d8f91cf339049721271 100644 GIT binary patch delta 46021 zcmXVXV{m3&({*gyw(W^++qRR5CRc1vY+kW7v29Fj+vYd-`+UC+>QtSowYyiZUcGmF zKjhE=B%-n$I0Oa=2nYFbqLo5XcRj^`v?3HxL-KP}lSoO&IAinGakGQ4L|n$3m!KT7!C>*R^ zciBxgzMlS9E$!>yaEuj?b~s~Qx92)ymE2BzIcK;XXA5n;e^XzGtIE$ahIatVjj|OOOIddUrYIlfl53iXw=12r7itaj@RrZHRY5BR3FW0H;V`^^2g zqH+C6i4}@;FS+Sc{F#S1DmLF7y2$ZeKC((jOPbeDevW?wKhbVF{#YIG)T&){3%;*B zo>o&4!WV_amb#1f%4TYwbKnCqsklhrGeUm=gqLIf40;v*?skFoKh(QE&)H83-N%b$ zMCmMd!r%3+Swb!M{JF5K`?D36>{0hpw^~avDMkua6XR+_@4AC87(!;b9=mU1H{8 zJ8{ge{3H&MJ&>GBUPv2@h?glCS$r5n*6)B{x-|I#wyTf)LXr8+(WG|vry^kCfXX{U)URt_7$bu4s(esA7`;}f=vyG;BEspnWI@VVhsEeO|MRM|CR%Vq%L z2N(xzckA&2>fcfp4CFcnt6hT;ZK-L}ly%2iU%gY{iezi7WzUvue&CYh5`SilgU_R6 z(L!{6tQVMB){EKT6uuX2x`iqKB#6}86&nAEs2kP&4v5)f_4AB|4Nc(?m@TvW$sN1yGbcbYw47y2iBz41eZyBy zT1;3i{(I$yY~|f-p=7wqq&xjtv)7zo2{-E$iFHki3D!GjbhBELQ z7P#|#e_We#mL|8@jPnm8a>oZ>K0=!ieRb_+(D8J>#!^>N%_t1x%Pc6 z^Xa-jz5P)IhC4^DJnxQ9JzwoaZknzh3RUA`fBxe@c988zbA@0|{TYc;18Q`c z6RUeU?9e=?P*h8ZkMBS&EIT0iHRz76Z+^a%T|;e!Gme@+*y{?vdk$$ktNK}w2vnQ% z75y5JgWxz~@*Ap1>{a3Td`76?_T8d9`-b?8+aeI$2=meIBS~sC^iZCfv8c4fve?zd7`lhW#aad9J}zy&G1Fm4G2*K(Sq`hH({ z_chzTGnh(1B6JO#x1V#IcllfS{*Ih!1}vX#KEbyZow@SMMz5%REnmm=>DY*;`0gnx ztk-9!QFPbhDLp>&88j+l*Dc!|TAM`mZLhoT@5uv>zF;OTxZYJ}qH*WGw48#LvwQu1 z@XvgSoAcknnXl%ZO;#7Xbi_+t{VHAzSUgU$jq$w9Zs~62chK&UivDIFI;)Z#Nzcbtk3?P94SofAG?kAG^%qG^ zM7|eLE4b&A>nL&qU14AwC~%FOcRg1fJt8lD4;Rw&A<~y@O2=d6{X>*$7|?H>L8%CU zAC4Fcc~0Z#Z&9s%&U(C3_=?E1$(!l&@MU9%?qy%Tdh`$gsZ8HBK#m_^G;E|V4Gg!j z;i`&Yx>KWO&%-Bki{G|RH{K`Pe8|9dL>vKJh?teBrbf^sc0w6Yp7|PxVcWcx#>Z8= zTcrwA=xT6Q?0wc}IO2^fOKR_qnzLee-j3N@LJDeV7Rmf%c#!K^`t>Fn*lSeR1b(!S zkR2}4ilL1@N#<`qva=-<9HXEozz@SJh)vJXUK261TaLK+T@NNC*>pvs)D}uVhT#BM zUaxz~q5*3xEY1fn_LW8OJVkE_RpP;_46%IKbgkV)Dfpa;VBRa4#9oB;tvrv zAz|r?Sk=>)HJB9OoTqM(nB)0~J3VQ`ZmazP==pOg_+#jEmb7h%=gN09R9E)G!hYxj zw%Ny!DrrOz{i}ho&HaMGpumx4CJ3*>13n({uWg^v5 zvtQFnw(6oU^j?4up1+yEx*i)Py67Z5%PiGb{Nqb*j|l-y#5TQmr+#HO8~elW$zDHyNqOp1{x4;$PP|NRzy z9w5dR;~+`ap=G|$mUvjUXfzP%v35UXqN`?JKDiJCJDb#)iaD{vu%!BNKUoYdWgjjD z=iTF;N6b|DaD03J18&GYFM|(|Pu}60>hZA_cB}Qw^L0{7=nupOAD=`otIiXqK7=ri zC@??7JfwsG^z&haA>k*(D14O*y(D?xc<{Y^-5?=e`IJ?2^092$o)=d1d5Qe_B>o7? z(^*9FIZ%fmdxytB`3!QbkEiu0zd$R^{Ar(*5xy9KP`+j3O5|vu&c^~MwxSET!FUX* zB5GfG0#b47fPK>3XJsXF|7=o3+^U)zvST+DH^S@$lqVtjDy?c2JQ_u4Et=IXokV|s zzTZTyIb(V?zMSzdue@YiIinFNz6fCGU1|3~2%SCJ0XAiLD~5!7vj{i{L&ELlS%-HMLKyc-q~A~ zTcqvS`~w(T3vvLx`c{KEszuAbd2s|2=$E~cOy2DE440}=Jh24HN5Eu?DGZ9SVh-d$ z1e>F^4LF1A5JIkfgTgKkR_%@RUbEa48*+s3)5&XMhlB_z#yy$e$W{*crmvUFJ2ri& z+X(QyZpSUyFO#CJ;5ns;pbp)^@_8lwJH_(|&fnesjf1+>kMbMP_uK#O>cujC!p69LQQWIm!I1g4Z zMoF8;-hJ{Tr(hq73J}?T+sPN2kV#?MV-+PZt_>pG!8*+UivDL7;?hqb!ZIR%}TEIV2(v8HvqW z;TuT|tY#jP0?!X-KNPfD(%bFrnxwzdF<4cZcm*6l2F*?WuExdbQO1mYB>)fSpQLqf zt98BUnrY!U-a4!~uIzZo8Yw3$Oa+%V7d6eJ_=8<9pch=oL$fmg|7^iinZk7W>Un}U0k~z7^|{F^QlYEqRgAtv2m z;RD#^$5w;uW7gM#MfQE-Eq4NbMlpPJAL^${spM~{{2m>;;?G-I^oDFwBH5~nJxeE9Ja`Z|CDR&Ir-3z z0_q7P5bMT2+42W)R?T0u9X~ghghyD-AqkMeBA#LAZl6yltgfDn6C!Z??byU$u%;^7 zynsR%J+`*FMCO2PS~7%isIZhN)G$a~4=$xCP>UMRaF=BlwWIs?nDH z)47Z9|MSTIPA2D8@W6oSr9CZybmke z(6_!1e=FGRJuT3R(~q9MQX%qMgOYr1Rxq$rtUjiVLjKhKFQ2gE?g1OqIVUIn8$ch- z&_YqiNlL;hIOnfdGg`a)$?|j5z#u@RyMK(n*M(R=?YFQ?f`7r^0c7Vv(yfWyyPrP@ z@CW-lBF&Rv>N?-PjmIhQ5z-V^4GxQw}rT=qyLpY}{M7PyRi;L67N`O`&j0H72H5=4Qq!eQ8^II0kSR$X`*ftUGkIpO zo$&W41Gmz*u6eV@E4S1<3mq!>Su_=YO|m2jRtKb}a1^4U<_wxNvIV7Er>n|*zlPNl zAVGsflJANdO;)UPRzoJ{_L0IDu~^A}f0}X84=%~a6+!iuPYE#xsG$xu@y>p2_k1g` zI#Bi3tA`@vVtn1yS@SADU{>f>P2Ora`yO!fQy4Zj>qeSg-?TYpn*ZSH+1Z*>x#+2v zRUF$8wfXQtnQb3Q|Fo6hTDGMe(q~U4y;B=GXO`0mzu)A@ueyB`!6e44{1&O|)^v(u z3$CB2wykXN8~ic@Jg(Yl(UzI^nay^JF}!ps0dYg{^nNxoy;jSKNTtnxdgzXisU={x z*E%@s9t01nE0cO9&a(`qY6aH~U@`OMX3UE9kqPE7VN4r83gu&JCp?aousNkds^WEv zS?!|FsCg@x2C!E|`O zs{XK^?G(HNR2+eT(01yX-BYU{ckWCH*hAMtOB{>$n4jH&R9t%rGE)xdPBr-W`M>1W z%jD0XbjEW2VfkffxwM}WyRy7$+;lu?@LTj5Tt|FKzy(uN(8mUFel0=bL0IT$)ByPT zT-lT*#VNrZ?*m+8`x)2KThj;c5?l8*E_6EWF8LE2;Dnq#%-&(pfvh<~v%?Wz*X|xu zBtWbS6P>0fm+R}2GX$iQqnj8X<4^Rrw!@s^39kv?~-J-+TLxeA2llgj#*gpTpJJ?^pU zJl&a00`3rU;P{aX7Z9fLm4C}CQ3q+ju(gv7P<)}J87C@{3b5LZdpu%S0gn}j#ZVV4 zPL->YZw^O5r>nguYgBj3;05Mfr&^n@-xYp>E}*&dId7NbpM+nk4qVAh;@!(=_bhVd z*LQ%-L*U%mHa;*ZOQ9{I&1PDCURE@bDsW1OWlBPg`8h=t4hSQpb*2l8%8bdD^B7(N zlQhV0zxqu);w0o4%C?RfqvgA)-|YJKz2Rvz zZIfo;wfP+((i%F?&MV22j6)@fp`aqJjVKdY;& zx5v4(pRijd3i0LmDKymFrESgO6kt37#+it%YnZf~85W$D@>*&-3+vgsar0#m44R;v zOcl#C;)J9nB@^YQ#Qr}jZJIMFo(go%o6_P(3lvE^JyU|tVtxRLu!-M$mkg@q{LlOS za(O#aOR~z`q@r3kmla8-_i@J53rai9mNpPo#oX1nq%v zFB)q~H;ZJ>=L`j<$>V>NRq0K!_ALz!^C{WJUzR&YHr9`}tr?%oKvN?gORv4F)Y}q4 z)zM7Wu5hBi3QzD!Dc{{@cOXat?(0q0I?KE5SAi>@;eJ>5Ui-X5t)YtyIxYKZlUa-| z`{g4(@VP}(oZ}G7{dw5~e*3v{FgCr`7ig#@=P@dnrgpY?*I*Y0h4>?RH^K-_&I)4`9c@|MG-Sd@t{`H>RRH#?L2^`e?Tl88fSZ>-4`^|rFK&;mKp9nDh#LJbJ3{nys4 ztmS=JA#S2(>_i#35G=RDDJB>g@QaG*kdQxJU$1$soAgy=KGg0Gamkr&ZwcF|?h2=o z#(pPaKdL8UXP7dFiKwg0_M~`x5G-_{A^(;%ThuCK<;tqtr0(u>p9pd8uA}ehFb`xb z@6d?Y4E1_31&EZE?+F$IsOFBA!E*fOtO`F3w2esP_FS(pHg0o&IIq2LOyjejS^Nhg~+xlBAFcFW+4bf2&oNN zZTc~Ie)t$$F>eSM{41&aM(^LcR3^=|oO$V2Nvff7yr)s=oZLHMKn8ZGoiW{F*;%Mb z-SZbWfHK9zWW`Yf{4iZ= zHWq3|YvZc*BHnxd+d`qy20O_!{53pKDZ0U}y|#LGJwP8IdWhy`^`~ED_y^vOor-pr z??3fiKoQyo=Qa`Erh=p~+d+NuX!^b-1|?yVhjWbJIRnlR2igR-Po`T9Stx5{QiZHZRsV{ZSrxU< zr$WK95eN!EML{^Q7HoT8KlobYBQj`p)1;-w<1EKg?wqD-6Vy(pjusLuRfuP5BTfYL zLJNlLJs|*2=J@~^x}vD3rCgEU1R-t?+WfDUkb;Oxkq$2e?}#(6bx0T3YN~Ta%g3|RhDKM2JZphYlyY+;~dT^9qy3^3}Kmo{W0Wx zZ4dLW5Y&QBNr5l*53rK{`fz~}6MzDYm(j|DpdpEG;POMBJZB{*(MtXwa<6gHB1)S= zN=FOLu$IYQZKD@6gWSNUkI%i9_Xl^T+)h&aOd`VM-?o_ymC~Bk&~YBCco6y?Ph=IWU(y=Y7-J*MKcoYITBnW< z1^-`w%b3qt>5(-D-su+-%`EHLpyoMxw`jyZXEk2KdJ)_?iB9LFKa==S=t+16Hgf~Nv-B%fB2Or3_`Bn&=Q#fpkt}Rl&8vm1J+B( z4GeEXvd-MD)y1O_Q+4W5T-%n)#=yFo zWT}_g3hWaqA2b62D6FuJa@la(61aoxW*E&JuZ&X;#uygb1T6-C_YVFz)xoo_XFMrb zarQ_uPJN|HTB~=ap=X-7i3J?^XT!0PdgzA1RgV2(CUw;2y-W;046G9HW2bhF*Wjq8BbI)}3eJNKh3l?0-s-4 z2n77aHi`VGnN(T431;&kjl%Lj;p=#>TOj2^9Uj~sNsVwekWcj5;f{a)pnr2gA9+1K zgbclxsP5kfY#a;s%(Kdi{KE2)e6Ox{Z%|XtLpDCxv+-?emGc zY$SZ2R0 z?vWvFyQj)oBEh3K4JW%-Z{)WBs`RL&i6@^ z3Gd530{mcgVHnWkf|g!%-;+P7_dl;E8rDk#2&3^AHCvXn)=)K_S_$nD+IYQ)nfeHu z)GKH!x%E%KZ4XGG%|~3b0Cq$Auq$6W&nPhzqE_eL?;`z_nD=S=6$j~yiBXl=Ln!*b zH0!ZfQSSBDoZk~SKG!|~ zIlkj9J72NLhI_WSziAk@&ph0<@kP*A4sI0 zVby)yMfQzFaVV{ zH?6s#zn~9@-4-ltOGYZWYa%`Lf;<0SK9pfHa;fy>FiX_*>U}1efCqk#EY`Ao++kDK zfM$=xG%%AnCis$SK8g=cdCv+#Sh5TS+u@0DIOeL)HYf+CL8c^snfIFTn|=fWGUX8p zeEVu^Js+vl7wM|)!q=rsSo+CB~!h{>{T1KE_i0INYtTkC=vE!=fh4n zy{AhvECj0vuK4&7%+=k2M9hbPF-HS=9KpHjQuKR7+as#3tiaYhNA(v8Ntte}-nKD4 z#T!e{&5A%Qv@jj(KyGGxW;E@Q!(fx)M>Zx{WOvbtT<_+xr~c;#XFHP~64KwWspTP{ zV-W+7yZw-2ZPxZ~umeiSZd-jB2`d)uz6jFi#1#h1Q^r_|N-?vyQ|+mwSFL2$*);Yl@H5CjOlN>xChh z?#2X4j76T$q+%j_q~`4Te*5wc)fE`fbqoF_dJ z)dg=)qSa{$Jh!?mg?eb!-a|IJ>y%i5b;cQH8UwZ;v2_0H^p(qqM#)?$a{s_V@6XA-$I!HS zwSU%YJ@Mt*Paa7;;{8$*2D>XtAbm9e&r1)8sByjZq_x7UNOhYxTL1+%||e%R-LXpw>4qhX^q?T#{A=`c`L2Ep0sra>43ZKRqG>o}c#~}M zcoa<$AD2j?Y-wWfvJ4ZZ%BXv?Ltw*Nw6IpeMJ1rDIEO^@5a|-q)VqGW&JLCZc#?*7 z8wQ0L2#~x|UCC~aK9N-`%$McFvH+Snoa`C-!_3}TjX3gjXce07G^s|)su!+`=UM_S zS$|mZg^qtJQ#0PW**um@vP2#v$CvXZdB%pZn3T1^QKm{=*Sm(GN$Qlz%E&{hi(@`| zl>mw*6@D9BshdM4+fMn>Q`U>r%BxH&^KlkXvj&Xl3dQDG?b4I#~Xg z4PvZZqUqtp6DFNUfFBT6W0cQ5+}37Y$&U3Zx94MIG#sSe+`f_v$;NuUA`bi9ehdC( z@@|6~?{RI-h1bkb4xtJo$oM9*x+h&5QU~}c*^okZ?y?^l2it%Z4`&(8AE_!`nefCS zt+&kgsMuKIfw6G~>sSEgyAtI&iifZU^}q#(swHB_`gs1u8vXXa7TzY`p$2 ziD0e8um2qV1#TlRqYQC3*UgOSlS^kb7(x{Rp1#|?OaH6v0sxEKrwXp8q{>W>wspMO{2cX`aUBsRhkTD?`KGlbr>R6pSkL)&a3|BEB*X;Ha zbT;&Hr-@M9|83ARAHE>v&axNI{W+HlWYZFJjjo9?*WUyy5p0R!;N7j?tZ`rmEG_2L zkgE{K_xj#&>0v$eROv)3P#j#|at&*~j~zwI%$}Y@5C#vKb`=wcexy$X4}cJ~pRb~k z8=TW*&>l$IbUt!(Rmj+|V>$Rp6URh_1ZnV3!AqpxyGFJ@6e-eRbtE{L>5Mw(;C{}B zTh;Beqhpxm3CV*xlY-bQmhTKMFv#q|g=#$XGYtk;*QljT{$wSrDFEStMau<~e#kOvIa9!_PBH0W?su09`p(;CM3*xk;Td zY%123vzitH1el-ND}8=^a1Dfo{QS-WAhSLwnIw+IQLF`LSH39a6V3WChC9z;JzB!q zJq4zTw6~|a3@R@I4DbaBI;4%FN8}|&1R11W%gEgPoJ=seqS`q60HxFjoj#Y)0dHo_ zs{8aH1IP|I&-i42)D;WcYG%0)i^I0g(cEtw)7=?=TPH#_(uy_CH*h^STO8%`CQ?O{ zf^nrjy-M9^WEMl^ilVVI`M>zp(|2Rf@bTj5+m(L}1^9(;Pi}w4g+R=$2g(t@K%hY6+ezqcWrhl6~D637CPLb_6~ z{%qsDG{O=J7;2ym8TGYcBUGUei69vSS^r5kE>$q_HA{;3qr{lxn$wyNX9X4N>SrX6 z8qE!+lqr3Ygqnd2jmJ)aIci4W5{g`e%dlC60nNs!?_XUwZnj8)tk-Gi&h`+n7z1Ql z8X|@W3z5thj5E&_D7MpH5MB~RkzU9p54L;Kh|6qBK=SWAoDHE!o_=e(*vsN53Qg;V zJW2<8jt^w&Ti| zaC>tOz-NB%*$U-eGFj%`S6csl=6Zfhlz(@9xpppvx4XVgD8Mm46OX z_adLeIRa%GdcqY8h3-(f5;S96dz2VLz0AA?#P-9HcLj%Un)|}?jH0$hX#FcFApitR zIjMwwJ%MHDN8*_u82~|9Bx!YJd8h>%IxF4@Fz9hi*Fee+GaJ}V*6t(%9dMw+v8{}% z;qd;$!58x-osr9j?fDm9t)5jui=QSn=mYFc&T7o`)j$psz>`6y!G9S)DVexz&<2-Fz5)p}<<@7?Z9Kf*}QaKcrAj2kz5>6au2 z)7zwEfJY+eV;Ma_?fSd013{B9qLau zGmD+MWhj%B{RO*p3e4Rx zdIE~oKH90N*!@0VLlFKkUdMs}kpJ6%5NX>PN^sJ)SV-nLOgK20rCauab(SGt?rD5R z#3t?alGh4Q0;&;gOt>0%EKa)0Gh%*0mzo!RRb0O(O+Y3%K<~QiM$00OUvFr3&&;c6 z(Qzt92kitzggM80$5tW|!GzxMl6UsViHSfy#KWBkZ5LjxSgC>j0QXq}bfJjhnyKa8 z41oK)4+*>`3Je^SYkIl1^g28CcFf-`K%4|g1HR+3X$fH8ZWhW-Sm+;V@7#2(BUJ(hMlOP-+?lr*rBW>eO2&Lxrs~ugfDeYNx1;GrMhrQ^zP*9%YlJU=iu=>KeEUa>pEggzpi_Z<@vFju(=(CagU zET>S_<{OA~%pcqX<_|D?iBWd8BOJ_a39IT1Ox~&W)k<2fRUD3Y8VH96@`9m1>&pJ5 zD?cWc8DUYUGXTWaw+c9 z+k$421vro6C$k=_&9FXM3KC}nWj*Ou+bS;zE?QWVUp{=e+yv0a$MZRusQbkXDUztUNje8Rv#>qX<=T2n6N0^6v6prXxi0m;}keRp&`N7=ra zN=_|N6WwQrjYU(b1`C**T-4MaYqEj?0!|+1_6f6lkW$xQQeza>HwfyA2RAKUpcShC z_4%b~G~b{(vd6Q}>N0Z4Qe! z)4R?x7xNN8YMwlW+`3+qPi7kl>!WB#BY{RCN&32w4WfZ{ZLotQOrqhS1x*@j&rcR} z7wjxSwG10H3?Tj@5Gd#ZH`kM~zIL_+IFQNRH{fQOI6sHOUJ~PX+o7!oKa{#hUd{%G zM7cfxz&k?Gju>V8LENov*)AmXLr~zEn=?OZ;UisjLvMNAivWx6FX5nMGpSD-h3aOO z!|7od2WvN_Lg4aQ^OGw9Z&PzXeJxY)I9JSANbpg6<<@N*(=1Zkl|6lBGwr_#IX}2s zGmBKh1=5}ZletU zszoArtYr1Ag%Q|c6U|r>*UHH-}5in&kXPej1OYekF*GT z1~NH2jz(pr$Wm+RR5Xou3=L8)z`w%-u(^_=4W|BHR`OGmAN=gJx0I0$g+(reaoc6eq%dI-fF=UxRE(S< zcCT|pxc!LL@)E~u#02My+sR8aUi_8JVC~<*xizXiPG-G&T&SObjRXrpxJbdBM>cfi zB#GL;qGTbzzm08l_DxOki588E64;+%`iXsh#dAwnPrcXq32z#w(upIIW$|+g_L?*3 zYIy2Y%3M1xm0W7$rTB?00*o&zC-O}pO#659No^5V1y$giY9ZGs#TC>xDO?S*CP{5U zfVL)aB;ttw)g>jYLaK#-1b{}whwb`|W`-uQv!l+~Qz}nY1W&S#6m-^v^{z5cP12FIq|JE5Eo*1JOP6CA^-<3R3&jNpeEP_ipOIYTHK#a0 z$G2V1u z`>?)EJ*tfxPlQNn8Ft4Yfhc!e?g4Fpz`7pFa`4@(kYlcI@O$a*=RP=I!2Z5B4o0M; z`z89%ji;W+a1%@$+j=zxshX`p?1OYP556RE!m$d71@#O*_e4GgZ`RX_57$cawyL>x zi!@HdvNwQjq7o1Z|NV@uO!+ijfX^DJsKKbL!|q}HfB6YS`d@mM7nK`AwJ|M{vUwEq z4fR#eiiJDhyFdGy8S(Y@!Rme~!(@0d1=#bCmM!S2+gke{x45?m5N6!)Fv0~Hi5>l< zrH0BJ#xi3}juQz(KOFt1yf%tT&C=$30VuJexIV{Q*J?7;f16ju;wsVdWc~cRB{peC zG{&X}4*$@lr{PI|q!%3H_}>2}$_Bz6V2)lR+y!rXh6VXRJw6O<+<~WS!TsV5s!kDN zz(-`4D5b4pelEPC-;@OP7i~o*Z%Snq{zv4(7paYC+kTN(zs!9ie#CCsu8EpD185P1 z`T{?8l%9^-6(bUTyUUZPc(r7dO%EdmWg|yh#D>e!eA~>k=}=!pQbLr`y8l$Ei&C_# z7T#naI3PL2yQ8J^GJPlg2omg*H0QhD)*>>7aw-sNa#M2FP-R=LK#-NB!!0*2PD;!R ztiZ2itCE)fvp%3^3pekr0-sa@4R~c`ouDAjRnd`1=JZ104L_cj$D~{qOwm_!CI7rt zVct(wzWX2j(>RZ%qp=gAr#fRdP&q`y%6K7qjF335XDArfIn9>_3m0IQu{q-2AOepT zl=5bEM7~u9Se89N)kG<-BiQ%M7F1QmWl$+4z-ZZuSBgWVsmOy>nE)J^>HZZ7JyoMU zJ(BRUC^RBBuUHywxldG>ns|&5{?tc~VD-XQl@t;>ltVR-f0O-q$X=Dn8ZLE>6kzKO z&MIl!AYCWOa74EcG?p}Hl=7j1TWj8QQ4$E_dws*qQrjHfI{P7#K?A|lSTLB6yo0SF zV%)8sawD#~75T17OaXnt&LJhgrxAvarygNPwa{((l%ulcDzk=*@RIsaP&Vs`BM|{e zhKP#bRJ!F*r7l~mgA`K}ivO1-8o?cuShs3_gW#>ur++Q%w0CZ`LtEIJpwi}|wal|( zC08kD*NmtyR_p4NpHUS!vdC#H#MY=3QAX7aMz{VwDA}R|%une4(LtNJ;z->eEdb)j z{PMt6SIJ375HMG0&7^BjI|sW|U8pWxqFQbq(_c_HA-6RdH(dB18EP&l{3pxrez<4% zPN&$INA>oV>wXnNvf&!!nvrGX^Hoc1mb1mm`~9=xkhC)-`##QGFAQsi|BY2&mHw3Z z+96i}P=r>^V);wr53=*Aeo&W{@KToz&#>%=-(jnuw$jV%Y=h_~^CY?(P1DO->nAx8ToDFrIG4hFv&T&rLRe!~d7pG*(kR z7h^h&zTasrcfgd#Dv*{2-K@JDOcoXmcsBro735hH<8mn9piZEADbJrDKOd|h+5SiC z=rUAAa{J;*a25Y5@8&d|;#NH$1~s?<)n7_J*6j#vX1S#2@<6Z;eKlCjEBcN92)=ay zoD!_KJmg=$MhS()1ofXivgp}0B&d1vy*k$;Zf;hmyP!M8w0cThTCTZ`bo^;U0NaNeBr?e(B4RY zIjEA2^Ddp&9?%61>E)1PPM=nwEU4z<9(s3Zs9x5K ze*u5EHimEw+9K^IklYWTG~-Bx9Mi#xmSd8i$wFr?BO+RSM$&t#Uq>ih7{MQ{C-($r z`xUQkMyY}XAW?aNGmk6$;T`EJ?A_HrvYCVb)^q1si97M)3D2 zb8|R*$&n=>6FsQ%;hRkaGaVRUtts_6?`C%MQ13nihTADbu3fn@XBKxOEV!%9?IT3( zv#@~PI)6>o5FC|K>RF913(zS;KpSG8* zfIZNSOtf{Sb0CwtZ_tfaVD*Sultok8W1Uwnw`9q3z9OzpY3ewKK!Q6Z1AHRP2MDNa zVbywQ5V;(vf9hs(Q05fFYd?RR67D97z;2(fdCn(Jyem zh&ckHiUM`>(V&zgxsGie#<7GUjVrK@V8F_RJIh1g^g-2CbM|&Lw{M}9WiSGR)tW!L zS^gn8QywL+oU>GTg#i$zL?d&?c6_ z^r?uBDp5@`{;pSt`@;MH{L=@#sj|CY?d{>DAISfXSrr*YpwNIvxr-z5cvubZ7tZ!S zP&*$0m3p@;LIy!Y24@a!?2fc)DQV-RRg)v^-^;DM?#LXLwE?u0bK{yb%CK}g0@sR< z2ug;M7L`_IRT-+xXGo6-|JLdpHwd2y*D9hCOJ^Z!<+!Xa9@mfHM`WC_F#!;D<$>7E@53k;xHKPwdcC+Lg!!~3@ za~u~iUA74o4OrV0J~R%aGf%5gDq=)jt!^%^Z^Jybs8)3vXX3_dTM{rgGctg}WXD98 z4o6t6F3)e#$F?p_Flgp#t5H}=Fo60$b#9LRKXu+f|DQS!BlgDI#s==&-X3iZ;T-|dGO0_bA`76#27rG+-81p9 zs__;xvK7Bh{Jj95>o&dEvtzP>+#;YFT*}i@)*5`e*js?56k$ckw!7Lg0O&*zh*X=% zGM*3(JD((rk7*)%HQ#Eby6>(wQ9bmI=9}WMX+^CjbsP04)@s&ONYyc_(@Wc=Kpn})b|}@>QTXZ1BWoI6RFJP z|KH)*_~&p45OHVx=WrOTrDkM1s;;`|Vn85APu{C5!f?bg4(rko27^r`ljk`CAo3v0 z!?&ofAmw(a)%Vz;Z^sCO|81DgLjC2ONe5d&aZ-ckUbj9BP!>&wRoUXyCw&J^Fc@)D z67eyy;f9-^=mI)pWx7bwFnqrqAN&BeGuJ_X$;c`N6W(PKnQ{+|{Y*z50FWT%deBMf z_{97t0lP05B_t!`K1Ui*iL+9Z2xUEKR!PlA5=4Ka5oSz+$nqw-!46p*2?U)s9!Rq7 z)}w7u1!)4f5=*>w9L6_1iJ@C5McdJk6D9vZQhT40ZG=URau7XZA}hJQ{4AW1`Irzt z&1dOBOBa4?vXeOtL(NEMP(|tr;goIT92=N79qR#JquMcC4eGP^X{M*hX@J3WWkK}7 zZ%tcb`!Z6zF%3-ji#VlQqC`NOIP%~gdLqra^)3P;9RF%v-4L`Pak*c-?tTiaufWP` zM^O8GcicZrHO_+TqJ zcSa73n~$aX-h4%!P88gHYE38K58clImfgADU!gli>olOKnW_zdABdnN;9qF{4&Mjc zm%0YPiW(0&q@84p#s@8Ry6?TSU%#E5g*v9co~Zr={y(O^DLS*RYc{rR+qP}nw(Tc2 zI=0P@({Va>I<{?FC-3*)oQrk0F4o>-?J;Xs)vT=vOH;}p)8AnpFLh~S#$8-Z&2jlZ ziz2EBe$MLQ!V4jY0Rz{Q1MB?bj#KPzL#-_|hIDy_iJ9=wf*5tUm}q5qSD~6@R=9!n z5B1ifP`7$&_g}x=pJMXQdz46fhXn9*58;x`b+v2dp^MFcf7sv@L7B8D&K9M7S{aS1 zA9Q)>al@34Uu0(fzfS87i?Gs*AwkoSqWHf`1ecr&lQ3UY0Fyk$D#x+wjG6kj>Mc!t z+bHwvXg*U}T9sV>KekuxNIYuJ%TfYE3LS$k9yZw3V^2o;>P0zp@Y zBvN4KGA;W7ZEA{bkL}A=Vm(u#L^S-`m>X>lp}i5Hf?>QL_owF+`LK*<(tF0idQ;|! zYV{r()BU;2f!w1f4={x{V}|WF!PYCJA!J}gMVVF^U`+NEj2<$@Ppmf<>Q50$Yy7Jg zu^0uS)L;Rhr9Tx7inYIZ6cHL7{#jCR&Z4tiULgKG5#8k603xD5rqG1d|M43an0 zE9bqVsrMj1L{&Y=_jPPo@14DN(T-H|IqKRRScN={z<1ya3BZpC`Ha|0t(-C?gB*c+ zyNAM%$z8`7FI8kQN$vwT&NK)>ZslkB>}!VB8II79|G@0JP81B%J#pCiO;=JiCb61B zr;S|O4dVXeRCMO>e)6HtaZh?wIU^acn(xa_U(~QO`g@suUJCY_T94*M{w`&}P)d-L z#;z!7+k&b))_-qU3f>w2{?z0yy52HW!- z=1ftXe(}(Q6HhSzPI*_1wNnc)+IHUdMy9Js6Tj5l8lG)-w{<`&L)m=3=lDcNv}931 zM+Eqrt)8qzH{780I!^WZDy?!-FD-I?E>SJ1SDNU50<;>Y%hY*!n2)Yx_!%mB@D+$! z!dPJQ5rwBdb)HZ8PMm3@mv4<)+DA+m%l>bQhubGFp26b!S!vQ%XXG{C09J5W_-1jp5O-qpm64cdHq(eM7u*o6}zs6^Ridx5$ z?MBqc>0MFkIZb^70T3D#-w~omVL`N~0_54r?=BKzXZ?{)W3F8L@Z@siYU$_vG?Q=Eu1@o%yqKBwU`&QkOOUr>V z+qltWg9iF00rrB>FoKQ#oo>1yr9!y5@wzg-Y~Dn_D-N6_jmVQt!Fa)SGadcE#beSO1T7UYBi#TXgl|vpm9`5r`he4_B1uGZ+LMS%7Mw~PH-$7 z_N*0v-4$mEjf&*3IzC9h@tFcpfWNBS?hVz66F5xY(0y*^NXJ0EZ-ybQ82*hlal-zh z6oF<5llgC0&Og@;#n9lC*m3j}`~y!aI3}Cg!(6KF&fsK*-k7za>iazR>C!a-&B21` zEWJD3W%dK=0l)XP`{$VTt9eyz$jE@#Kf;OEC@9uSVr-5+#yhJQvf7mHfE&~|H_A}< zQ)pxQDd<89(oaar(Cv_7g9uy`(+O=d0#be^xapwyw?3s#34nr}KguuePbK{PbO0?X=W^7Zzpw2^79f@_Mt&MH#oOV-$7lj+@z zQ7$GW=l`GP?oa$}opkehdb``+%cv(nkK>c=Sr0PK-(Yi4p}1>K0rU08`wIRlKm0iX z5r*SgN;)i$_kp-cavColD<{KJch@@6D;mxO8Q=(rk3%IjLxXd8-+mjQQz}qWFRT(- zET;Tmd65gUZVkX{d+uf+3he(!_Yi`C0mIUyG@bd5$2t&B0K*m>uIMj`q{)Kv1+jL0mhu&HaLTO zI!Y_KsJYrp9j{5TFUzTu<$HzAv)$X*OnS}-7F<#O)^4DyI!nTw zzt=9sz0xwX-d~A7@Ys4!{Mz4mwIZDlj3zO)`ZdVn8fETXlTJ5r7@b-xbU^7ZKFkBi zKV7Pz)-Sf`U>{E!p;YYGZjzaD)ALrXl~QuiXU!+NWyP}bdpO4qq$g4Y9;U+Rz$-iC z7CuHZLeGUf7@=7#iZaDEP8wW+h7w$U+%?=wJHa&8dB(+0Y;sRe zu9Ye?C)c*h!vfPWy^De@G3xQ0VKoHU1#G%t4Bl6M$q&M%^O7&GE_&o`S~);e<81bh zvaI@5@Efaep;Ke12N1?)h0l9(QEt(YW&FdEmFlk}74L8eqASxW=Lq-b-I z#!8l?XcEg4DQhoFds4c!#2-!f=<@oOWic4u1KX!7ILz-jhU@eHT=3*shaj~hat>mk z3JMlUcdUFpxRSBU0Y_HL0-V&Xa5*5v_q`}&F&`AHfVU^bK#n0%aKmh9$Oa@P<4dN9 zRn>zu(Z4)+kX2@gL;|GNwM&pmE^EU^RjiHi!^B=iEEoxp z4K2_~IK2PLMc$agR*j!6Fcs<88wpFs;&c5a61;VTup_%V%B4K?06r_)xuwD~x(V4Q zwzm)-dbUgIrkjbn#Y+1}#U~8Imi2FGrwr%dE&HP%$5hiz6q?{<05ky*f7cmoANnx8 z!IIG_S+TiN+}AOzLQBc4<#<0ss|EiE2e z`YaGgkb!}9G-@0HDh%)~(_@c_S93inna9JyW{bG8DX(U;5w#SzU)hXvX^5e{2j3uV zMP;xx!(YXb=z@gmHJVot%G(L1&g8lB-Elx7SM|@Av;5*NN{VF1Ld68xDoT$W@%B0u zYa0uOhYRfp`>e2Y)-tuc(2)y4pMlA-^q9Ov=0kp>ffD2qu>=^0R3YK=+39hBK;oD+ z(T62+$-r~cIHk6gseS%kX3CKitud=D2G<1#b!9CxvwNC0xvaZX3y$}2xWk#|8{?s<>%8nzO(!V^sYj^({_9SZzzRfw%sikjVNGYdebA13skOeU*=w8p z!R#1zQ(otO(Q$WGIPj)m`f5(R~ zbZ*44+z#M)@?@f;l|bhCeHZFyZGIt9HNNslk|2s(-O?b4c90$E<5FG(#}j|4#uja) zn+b%6`L{hs;#VDd5p%4!vv6LX>_Y1zYPC>Bcw+{>i!@^QQX$*f9p~LB-WMLCygba) zf?hZRbE9y$#cy0T7orbl+dRmvsOOXUNu#2hlraDU0#8EVymgZcm?c##6p{=EsZGXO zUyli!Ym}g8SdQszxKJeffhlcbB(zosm>!4#E*>276WmK7_C2A{`NNAWezqQ?G_9AN z`K36V;|Wf%!ui*rF(lHMyD9H|PWP{E6t<*eQaBP*rG@LU2j1PD1JS@{r))zgN#a!I zE*F3rUDwoc0J#&6oes;6@iY$;-&JD&bw5TU-Ghzj9~UOZguNPjrB$)oYfX@W1Vt&4 zs6KT%J&SWp+yOeBVIto_~e>qM3-p&xV|xeqqynjd*4C(`Y> z^Q=OzC^KH5@(>X|Ho>MymoPY>Yd^j+QQ;B?er2o+GD1d=E?~zlY1)u|@%6E0)-Doh z+v_<;p}Oa1=zsm7ez{IXi9IP&co2XPN49rdLZKPqR5nf52SeGU_DbR_Q=p4()==7^ zb3tsX-vPQ@^N(^W0KWT>4=VUf79l5HH1t<*7VV$kwXQmB7TRwAFvi`Y3bi_*(hO`n zohdl&JpwvVc&xBj4}3}ps=}oKtCVT5X=BHCS!-7 z{8u&QIBH)Y#q}@}&YkY+l!DbThG6(njV>$QUDDdK=K%?Ek z!8?*b;QQ=BG)`&Vb0>EuSpon^BcO9>3ip+49U0=5P5Y6H^}_`$?gxbv4+(mTVkPK) zIF&Rn3$T;5Ry7*;+Nh zBJR4zeWzno+LX|!q=JVE*wnN_gsNavufYVCrmFv(yqfJ!0d!PJ&OEg~(GI9}N?2i- z!z^!4UdOEZD9`NWO>%HWd}|9Xnl~_w1_j6$0f#aL z<_!0iRgb+BZ%(Hh*Uj?WkV^I3pkf>MWxstPr5Qo#HYcUGw42BS=}55b=M8m1Tqb3U znFRfYR(zXwMr1FI%AHb+%8fH2y~?VV&a=U^ueom$h6F;$UI)-sVHY|5%#YoAB?%@p z9km2joPdlQPn6O%fR zfC9|41?UxBWEF+7DiwsXz;VZ`M~DcK{8E$t3P1~A5W2D;&U=d9fC>(6G@^XT4Zwe7 zf_njH&w*xHRQ;?(i_J6RDdWiw$0vz;&Hn>c400Dl1?Qe7>uSG)!)i99MAeNLVU+c*wn)aNx`&$T`lV zGwfe^`W&jc`mHQ|@eOs{S2$hL-d^qKOEQXjKToKq9~%o$n-noNiO`KGto2htQ#}W+2${+R(BO=`ua+uyPP-&Y9x(640l~ zj9+OSvCGn3dz9SRuB`5LAU!<|7yM5O3(4p$03tSTOvRX;?O;vhVN6l3;0l>^p=2VM zIIieOcxsvdiviIinG~=m+dTt-)7M!+toxcV-GfAY`MWgc0pBZ@;!=X(Y{kHQhb_iQ zd^g*Wt^m$Tm-e|@Df-%VK+s84C?J2SqajIOF-x5c)WGx{hrYZ7)*aD>N*K%A)*Y%n z)h%CyXiQ`Uvf8YJ+?n$={6cs86Pj0q8Z(y@(il|F6Kq6frgr&Rt-ze<7rdL<*~Hm%9*lVSnDZ!7_s#R$n^x;tpS65I6o3$ zFJel}*X@s?vxQL;saIG)P`sCcTMnEM~yfu>hMBeyZLK zdO;Cg#&&70jIuyp%!XKX^T95b-Z9R^nD?B0)466%sAhj2#ut#oY7jP%`>59{+Hior z)Ua9&LLqygJABb>KCAX2^AE2`G`J`;3Azp;FIV!$j%WP*x&~e20W5eOh=VvB)8&Mv zfLXIx2uE#223;+qwou_{;9$una@WH`xN&Te`Eo~ppnfLgDu0CaG}c@z!7_L~-BAAE zm}L;8J<`!@ey4oYqr?ntI=z|$q@6{t7Nj&uSGjf-_t~xP~l@`p@){Q+Y%*#1V zE}JCpBwZys*!fy@sI{!@lf*}7I`Tz}18J;Hf#SKLGCYs?5R@({KDFmhmS4@~rf-1f zGw@M5uh~+lZf~WI`(~3^Sh7^EPos{j(RYNu}r8Jg4 z$I=@>3#s!%Tl2bEzM_-~0&iRY8QR~yn|#jadV=W(Ex4rJUG;+io9ZGLy%e7&^oQ^j z$rz6~3l$~<$Ks+M1=@1j7%!qth?w_~;Y?7Yrp9L+QXm)ew9g-ho{?OP)-a^;D;fg$ zs+{jvh%hpzK!;cqGY#CAV+Di5fFZm`jfniMV7ozy7~6F?v{;PpNU<#PlGWL*POa+K zt(4D=lB~vLVyMPBuqy`q1Nh*!#TtmAa+8zVYn^towcP zwL3qTY}9j@SnUO_jaDfHY5Z~8(1gM=r6{NVhv9JSUTMUge%jE_?6WOOu=yKsUNrvK zPB|tZ5^u-Iykd!iRgs8%s~KGOspa?@vu(4y+)BC_8n&)OY(~sstj62srN&<3?z`>z zQa+A!tjDml=zH}F%Jc)s)b+|PG~#K}MS-(k=-MtPDz4qUvQNisOf!jjGU|TQ(LzHy z&2DuCB*fl)MvQZ`ksz~oT=NjHe@_%Gv3t6^%MbPH+zZq5wmW#Mb^-#bU;aBQ!hiXi z5TSG*f2{|dph|*^ul~@=P0N;mtm<-S`jK+6@Y6g*OK)Of%pSzEQ3dtSQ>TXZZ~jKg za?Wjf^~FSPnIIw?iyVhPd35#7q&?Di$F2Q5+Lh=U@44sNN9!iFNcFNmp&DxQCz{ys z2x}33piGkA`9Xl8`90(heQ{R5IY_oa^ZYN*t_p?fv(Aghw+tN;P z+%43`eY}%^nC~-s?^5!c;@8l{?r9j#W4*oNvQhePYw4KOogHP(|FPbS)Hv##FgsQ_ z_iy2j&=^@}qv62*vN`4eG}!)L7_3t@suZYQrfldo!R&NDP~~kqmGvzxW6Xsg>0LX2 z_Lf#j#-# z1)KKC&s7>h3LG&U(iuzA&dj+n43-ExHe zp|Q;E`vLAiJ%iVq75Klh5ta<7@oVmTvdJ%BH|ou2mLDxT0Y`G4avDxuo{%--Z3-C0 zHq_els;+xZ^|$7c71A!|NgVm~Jq3QF_@uYEZWLLzXsN0*owH$i+2E@N)Ub{B@ttay zud$;E>o5P5Be_J63Doc%>}M`yPg@bp)I!=6Kq&5xX~^^hH+tl&9!!?>e^YrvhuBE3 z%5Y?r&;upY0Zgb+gO(=BrEN$!CEQ`J-+#BZBxf718Bh;28_)vX*n-3Ox7f#{&&w|B z{A<^yxJ+%7J8O}@KCj8uCV^vDO{8PbjYX!=<;mV28Ivcp{+%KK;^x07q2N(lD{_o} zT;<=lP<_7DX5iV1ZLBxg#H-65g!AV7LX`<4|0r}r12{!_QHtqX?Q^erW?j5HrK#;9 z3HfGFvJnQ@s^Pow+D&leQej0h-ARhC6Ei9t&9%ffSWBh`5iF?!GbiQLh4OXCwt**D zS%s>ratRlzJYg#DWwaxau)FB{?*`7r*Kt)iSTjOUVR1Rt?L7IC6-JFAl1(&3D zL=FKB$;Sg?VN}=&ZN;EFPuOjVIRn<`xlqQ?Exbl2=TQK1mPaiBuG3PEd+4`T5_@t$ z@#>}_a|evAOje~l`J(`vSeYwZMw}VPl^1yfpsI{53QO3;aG;orRY^l`X>3`&?-6G3 z;ZrjVzng;fl~7<3K~7bon(Um9Ik0@abKf6s*ax}jpbocY6|TDpwxjo7@9043LL{3o z(vhGrF%yol&`%drPrjA5nW$AcDoG3)Wd)wo8dSCgT1y^eIU3EXp@34QvaQkU7|y2v z=)EzAbWYQv+}X^H{n(!po;gd8QudoGmEJ4pVv=QpECiw=2b`c}awN&;0|_Tkrnl|y zeE7pjjcEfxTMLSSqWzm7o++?@M{F-{zu19pl}m>;52n5q>Mkn8+{!WRHw1X&-9ueX zX&(AwmD+~>sTzZ^TNcW0J~#mMfG@rPU&WrTG0(G0O>woXB7|*V{j0h*^Um)`nT#YX zB}Amu_Us2_HKWXaFe7e`aK+z&s)>%T4j5i;%CN^vRQm>UW1%n@>^YedTb@Aw7RoH1 zS}6?80z7=v*voGC5#~(V!#!pm6y)d5je^uOwwNDLP~;r5?oGy`%PbXr)xER<@9Ykr zGP8vW4j6WprF1JkiAH0W!D5!SKq9{A>qgxRE3%B=9^X&+VhQIKzCl1VN{_KFB#30+ zNZf%-*3jN;#o>DvHx#z6y*#~y`O&?v$zknh%0z_1jt4y~p1u*N=gyC^t?RzsOXIwL z8)LkBg4}iTjxSH$yQjp*r`_cM_z}SZ&JXz#32xZ~r#q2vJ~8Nfb63CD9w&{Vn3}(- z<29>zwgI>=Jf-}$X6Q!f(ux*~M_TS-fm3g><6Z2@8^RqA?!*8I@8d zx9!NR&5=o#$m1jrpHlvlB+X8v$>JyLVFIp0mdKyed-uNu4oPI5#ZRNY?c6Z3a%tvs zrFjP32y|CFm=hI9*h~EY?V?cA$uhmxB+7>HcrtA{v_T8lbK>XmyHQSdPe@mFqV5^d zF`d6M^eU5&=>_W_YtX$4ow6>`eq`+YeJ(0FF<+&bH+jsG*P2bR>~TXguyk(@PTk-4 z{coT3BZ;6o=o%-)8<-C54yblTunJ#D{@>Epb97dOZ96EW*zjTi&n8Ff0-r!c9D+T3 z#DgH4xog{=+kOGG34`lnPeAuqkPVvOK-`hiQR|KqZfGJ*US*G0)+i)^K6;8Z^#O-rzL02)2kYf&c zUpc&0R|bi06@LuqA%G66g1faqT1UKnGRKNYd%eVg_DBaP*Ol}Xdz={UM%)?|wWo>u-dZNb zJI|8l;sHusO1S9Ie5JCvrfX3LSV6S`mQS^;tscm{(|X-4t&dXs7;Kv%4ewdL$8DO1 zZjN?Cq!K+LQNcPL4 zL?i&#DW`G<;;h3qEzV+YtdF24uwSsL@1o&8Ru<@x;whw<963gC{a(7>M$@t8ncO=j zGvt{iHY<1d&ou5zwn6ePPQw+St2{q78Y|6GTyuRN@_v)OE=tP}a>$>3cTuO*_1y*g z7;dz)ph$`AK7Ys2hcJD}(5tZ2Rue2VA;8OvkLDKZ>2h)!V=IS#EcfO_dF=lo4G=V8 zZg93SG`=tRdWjboS5|J24QizI?Hb;X&-hR!u@G9*45j-@4alJyumYm~L zIg(U+n`by#qch*ZsUhGwPYetKT0*oD(l`4wiL^fOrcol4Dh7OcX_FA}gr^7yTaix{ z2ua5_l|f|Jg&z{)mzFab(b!1ziLN4fNI~39-B^+|wq#_$N02*SGMc3Lc=o0XJJx3Pqd|ePez&OGd(1OCnJQne)1c^O7+x zd&C`G;shk4^k=*i(iYSH=tfq>&txma;Y&_Z(x-`ZtdSxw!U==?)q4fN<=qswo<6gr z717zvmbB44#rxLDX6XCr)(q^ffT`*oxQ7`d z)Q7)>?H!Z>reLpXiL z-V}7*F2*!60>z@)0<=V$k=;^cdjXnR*-+69XpD@o0s{0wF;M#GrT`azF`0L$zNsHg zw7DS~D}^VDXcZp}5;7;Qp=F%v_w*Yl%U(v2RZ&YJX9DU`lWaQoa6D={fy*P2?C`R9u^=P!S}KMrff zRwGoO{NHEWBK{7p;!5pMF{Ov8lRFDA(;}*(=-@M{C8qE` z)G=k$O?t0d?Jf%^c6QnFn{Ud7tPzikI8J>uKs^&H{)m1~}x=t9;An?zgHCHs1iY)2Zh2Y-EJJDNktQ1h96VUIU&`_!LG1! z{0n@Q@ZJ#v3Ec93V5cPUgUD{eUTXk!hzvQc! zMUB&}h=vv9r$!L~3>JoW4A{&O!K_PUDktM!jtZ+-){#aC|6|!Ql8m=F8r!=!%)rga zt>ktM+mOIx_Rs`{e@tLbFZZAv1%BaPE2WSF4~HnT2rk0_Q`xCV6ylLiAJ~6P-q!$R zV;MNICB*AeUd996o}I7uK-_hR*1rKvFB-J$mQH#6Wqrb(A61<<-f|0cZfUei>(U7S@mZaz!@Lc|O!nI~*W4-Ix5H$m*-2z3r=aWw4@x zn`hAD0Hv3w2vwAtjmw>Mh%d`PK3c$vrmexwKNE=#;IDJGlt2T~Z;s!%m_qv%6gc?G zwlO}s&1Xf67?Sm^^v3LA=EY6U3m7d~6)gs0lGDC)DvG=-cjeHslE8|h?3cB1)mUFy zj#Svabeyk2;`3HV@hNr`s%bKhF-NkFV^pS|=2b~hsjhp-QTV?4`T*oU4gT)GeCEahaa~}UZ!jis&ni*v$ z>a+ReLo>AO4}V}$n3MGiCSd}^q_MVshkKGUx{M=z15sK4LLbvoP3>prP+b|)w@TOX zd#eykOl&LP+lxJgEIoAlrlR8(x!*WfzTd29-T!ioIE?prUh`%@^(vHIEu%!^P@U5U z4yEX@Zx@cnTkWgm=o@Sp7zW5%lxqx@Clc~#A%0aRkr55nx>nlw{Y+~~ze4%2vag3? z7Hi&upV>~?x8H=3j}-Qw`eoFR;&WcL?w$t)ciB&@w{owyjx^iH92ck+_m+S6LS=q# zP4EYcJ8GEbvRAN1ygKpp`c5i(CyYNz7P$Qtbm}7e?KbugG5od6C_s;Dqb*hF z$Z1<3;{8k)@2Yy)y~~j!tzKq_krl)@f~#>TiXN+Z6<{{32^cW_jx zKoHM~+-cc%oQ?_L*?;-Q`Q2g5|FPYI5@{$WSx45r=s{USS$gukTmWLO7$GGL*R0so z%I%ruTJGvno-Ns~Jzu1O8Orwo{NHU zd*|@Fx|myB7SMOGtI^5dJqXcQecO7Ec0jfj+wm57VPgXL`38=gs_&P6ZhkbpndUJ= zI%2u2cVjj7(LJ0s!6p1J?oJ;-Q?>2m8;0e0vzv>%fzv_h$JwUwUkuAOONb&{FDto=9KE@dp?=H$ua|89 zwtM$yHhKUk>agUxvUFyG5OcCny{5muPW*>I3iWlH;bvhROS1bFR295l{lQF*Jjy)1 zU%P!T=1NGITm(tL2J7izbRGFFKt(?~bGW3jJoC!EKE|%pU+44f7XcPlr}5$Wt(-?~ zy1V7pE_k*GOL(2KW8hp{6n_Iz6jx~Bm5M4Z>Zme&L7+7y+U@+g{@1?dX? z|K7VNTM?E9to6WHkGH}RhTdH${d@ErP0`|P$sx%jj{in*FZ;UcvSD4WGIYAD6zUb9 zsU226Q)U^$=_1H15Q2@xC+T-VuIJhvfG#c-J(&N!VR$IKNAs&#)%kw^IJ@cmo_5^n z&($a3Q}X)gNj_SDujui}ep$eLH6farN$Qra( zyjug5av3HG`#5H--yHprzPf_*c$@|&b1(YfxGW6NcIR?Du4Xt3qRey^p|KNd1&*sd zw1_rbCA~%8n^m+pey$E!u(og2W?6La69R{(*vdtq5IZmUIR>$Rxh%s&`i0My0GbUvv8BN_V&6Vlw zMl;rwxYUSh3kRpVY`JNcnYJorGcVg#CY7!2e(k09(b4$E#sgPcJDuycF33f5j(+oH zqdZt0qbo*i>=3{}`#3Q>J1#Q;;1rh0x6^Yi2xord6cUE$R234H*`fd`y)#D4_eI#& zvsgH{t039?+Nf!#iMv?L_a@m0_;cXPIh_v!i{w^@KzF+O(R5n;L^Z+f{cj*M-SEk+ zt&)!@&O@5})|f1Uw9EhV;Ll=NdM#&`%DQ#^01_IaKHi0PN`VeAHOb!hOnTp^k?6M3vNZ$6GQKx+=?#4D;0g857C^93Va3e zHlVN4Jv8o`%T${8tGnk|;5d__v#hMgLz0x$rBXI>+c0Nf{p%jxF@^w?OUbH80R-}V z;V%d|mK>81a4(RQ8^ARz8tPO0@#0<|-Y!3&8Dg$e(1JO#+p7t{Knm`mJtJbAY?Hi{9uu- zDS8YyFm=f`iT5EBOvyg7^@+Rdl=tC@WB!O3RQVw7i6Ohj9pLv_`u`gc`kOwQrf zB4A6;pk|MI2)vTkMYwLJh}=* zlCS&&ggkTA;L4U@;7_J%B0Ye@52bFwye-86QH2Cdqv)_08 zoE6(-2Kvsrw$n;juc&*6D1=5u$T`}W_I@QddH^>+029(ZvtTIs6001S_+0a_D)R5! zk&!v^jmFHGSF5SA)bs@wi}H3E$Hx$o0IIuxkwJALVSSoUh^4mn=W#>CSiqaWuU9f22@TZh~`B zeGw}FUe0oSQ`^&OgidjF4I=3}iA(Sk8f3l66vbcU5Ujn#i z^*`X12r3y?*%vQ(v(R0FSXFc$)ezxcX8`y(Rgz|u%N;m3!iDkn21b*US;JqPw|~eQ=YW&ytEN>ya?Mzc95g-eZQon3nYOA{ROvd00Xlj&AQGgrRouX%nm)biY12yW1<%%*<6sA^h6vY(M>4L)~op76GXH zd3Zbf`oGe16D#DH$#+j9RZb1#*e%%+0wsLdC~G*-CLv}sixDB#C-xP?ES?LyHqbd%U;P0 z&B@x-sjub~BF$`iPNyLxvNN(A^I&?X$Tl0kNGwEOUGTKOp2|2g1;qT%gi}-HRKxGIL%y-QkpMqHWu3Gw zVPz{6`H3H6p7C_FME`6P8Dk~MSvO2fTy6k!RUzqR;htAV1umCqVU=)&+HT}$QsLC$ zwzVB@h|;IurEIQ?wgCWpV8zyyJX7R#$fV>m2mu9-h?c!ZE^Or3w97HOB1Vl@(do>U zeAjnp1S>Wnw>1ptc>T)SJX#c9H4wK$r4Ys+d~NaW?bj0*pK!m*#&Dm1%r z%`W~-kZ=-cWrg$Ui3*wmK{>=d8sL}LC|C-@yXwk!rf`+%iZX%mc|i6oHfUYgnZlS@ z!%8~TiUY;lK8k^y=6(x7bhlaKv{mUJ7nL{3I-u;S+NqW!xkD_CDMhHg{ZNCLOuo z=__s|5qwEl_!V8kTX_|f_s@um)N7BioZGR-4sCFebE>(8Ys(<TGgDl zY(DUx>mOD6KyBZ~b?Lv(Mf^k#Z%@_y#;r@=D*FqlD5>ypiJTXVzS2{^rF$s)h%6;o z_KQmXjBw0vyAhpR{L8|3jw_3WNL;}y@%dB3>9Ju?Pd1BOszQQGxf1_g{L z8pU_P-l@s9jm5cKK5sIW;`GKQP*uKkt+MCv+uJ;0IWR|QtJvi;k@|Bp7Zbeor72ST zt!QP|dKP7CT2n7O-RHoWtC#-|@3kM`scquEX^V#uf9xHdV(WTN?)Yg^?pUnpR_{3d zh-ZNCGca?2QnF?G0R=bXJhm;joM;AdS}N>0|6I<`vghJO@3?EIC6LfJ3(GyIJ=Zbp z+a(d<(b7f=@$gb%D|npd#1d&3a7YayzlVoeL@*3QI=S~8DVv`QyhCK_SalrdEFd1x*TB+F0yt3ij_C`$b85B!+_H8AzDD?QI&HzpZC-Knq30Nh=ynOkuJmz`%F z-+uhStaZYE`88QK7hfh#`)tSo!S7K=G5FmiPqEf<(V@9uR(>yEBP-`Ki-_$ zaB%W^7Yj}TS6bbqy{c8s?b3zE=}NA9enn;4pn8yrS%O!Dehk26B^Sur^GF`OF?3>Y zqGD+VO3qTe96$FD0O9x36CC5-s9CipJn%wQ-GotGYT^i9?xE50oUc z&XmmFpDm5(ZUrFL^?V^IAamxs5wKc?@JM?W~$@B;~iyZ8b5qC(Rq08h5bhM zo4c=WYMc8$Br$0cDgBZr1+pa)Htz(F%&aaJ%*i0c+jNpx`tea2v*F>JRH}GkRatel zNFR5yF?z%|s=;8qErqPD<>cm;{kAz$enxNIjhG1cJOuE|Xcspa*v<6jL2EMbn5f{c zYg}9FZszGX*blWXLZYII`W4NeE(}^W7WSx#`Bjh#@DhkG?^9FXN>wdLyWg%7O1{y_ zBuwRu@Wi>}oHu`#-7OwYy$W*c-O%7HJxh4o=lc%vR8$~Dd#+f=P&0_PA?~O-X$~RI zc%!rmrT|uM%PJr{*L7?-MyEP#CX{pM)Gq>34K)2@i-+DStj^gvyOArJ)F5uR=*h2T zZO{W3A!HcFSS;)xP+Af~9mQ#2Xtm=1JqMk%qGrURE~#8slsZg1Xv;TbnXtEz9*DL< zFWXszdR||m>qnyGw8ht_uWr-{9_&;O_2j0fGnD1h+$Y-sk(S z_niGRwfEH2)O7Xsc6Z4T5>>6gvyalefqaDj<`AI@jSD~GIp#CnPk776tIl9m*%?Ks z@1b~{!>49YAWI&4nF_ie=}zn&uL%aG{n+}%uIsojV@XVXDk_<;&AR;OF4X)8>f!D9)XM^^Y4%#2(jy|>JyCw z8~(~b25uG$i(iw0_yjZE&K#UYNXE+q4dsLLUX0=?n#=p+DXC1q7Ul{2nZ9G#yINaW zLq+B9^_g+G!=VPuKzE4<1Pr0Mx136f26O&0DItw=&HpeP8He_+B>sV3f)&DQPaR{O zChgWDoKi@1HEICl3%K%EVEesrAEDv3H|)R^2%wS(yJc5_bSC9Pv!LU+x01jc6rs|n^itAF|#VY z(0tf|vvrW;)Ix+=Xr|U%Vb2uPgawsr43G%u3O1D0hj@?$;^AZqa9rj6LGJ^;_!1^^qGD*&=QEqW1^TP>)WBtLj)&%a(epuRv)zb2Qx1d2`{<4Hcz9=O+lAV*kMfgQ+n&O@ z000xy0ky<$x{)2E`Xo3IK>O|Sm&bPTOII=EYncgBP<%FCW%z~%bbM(WQ}s_rU8-RE z*-mDAri&Yr_Vg9)>44Rk!iW!>;;Qd@Csn~m9~N&;O}%U~;>JihDZaNXe{qvap!5cL zG7+iKSV|=Nx^tmW8KtF!U}dEhobr5`A7V+fmSD~mpzNJ$zDX#S6=?KK95eN2K{OPG zS5)r!3Ze;zL)yma>wvYMYL=Ci5sKn;zdZdSp-65|WP2Cle339J-_1ET=?U7$n*enX2}nl3>+IyzemNr99dW9p4iOh8 zRvzRBL=btwdAw(aa=`C1l+kA|aHvE6vf?R_Z3HEZER(M(C)|3uH12nJkK>S30h& z#|E`_qo6wm``Xq$Vli|t8F3-fW^GuutsWQCD1KTsOi1CrWvRtT?0|JgseiB{BH_}N z?#i^i3}}Ez5^Y{?F%>vVNt9OAsekYd8+4+D<~`^T_HNSXBuJN4B3dK{4U z0j{fk6H^~@YtjxmD)+SjTgfBWtw65RmoG!{j2o}Lew-Y|gC!6Wbkv5;)IM=D`ML&XCKWYiFOA^P?Y2ei~{Vcn64D@qE@Hi>efIP!J zZspInBM@R<^OijWq#ArT%87>9QfUdRVe zG)21TEK+Wx^Fw{_ka$+N5l>?U)7aTV`J*5ai8(2MHqKE}X|OG8x}55ppTF_&R3l+& zV}Qyg&`R4Hs~;HHZ(Sauqr5IWM?;EjPH5RK8(b^i>(YZE(3puyo$vpAoOmD!F0IV< z@4=dxRWTcew6KqdnoGzaf7R(gwb+ub|$xUCQar9+Ye&;6_|MA1~zPpAS^>;NB%AYSG=^swN zkQ;vxLJ6%IlHeN+x*Q1txffmr6XG1;b&VP@JuILf`5J8ND(r$vNTHA?M<20qJ~BBQ z{CY)ngA2#eU6;CG$f(bSIxl`Or7(R-d2*7a$tlS9xcYs#u0v<&&eWHAvFkb9X+Qi$ zeey$==3|O#^WG+w_sY)g#b>6vcLd?NBoe6!x7?_D=)RECB&LZV))c7V0SdGGlkaj% zO>$CRjY^>5sRzIe2()?C87#8~D{#Hy1Wc58^cr6vcF<=dC*6)^YGc zrE}ALQ7YH{@fyXg`Roh=CVJ0a-`{p;^6ixl?BVv|uB z7!-TyrXX4-HP}7r*S+rjnD54?!ue@jMgF}HZR^34D=*7i?;o07uS%&9y>$0}Y?{9(=M>U)M1z*F`kQ>B1Igz0q?beOX zMk8ypZb=L_s9r-0mF3|-9hGq=5{i4U2x{UIQh{J#8^GNaTJg`28(pSRQ8g44=XURd{T5`K>j`)=;=1_U|oc;{|Oy}k-{;iO1+ zw^_jS%6G9Hce1nQK-1pCGT!Oe3LSFn(VUZnR4F=n`@u)H*664x^)GW* z?DB9H+$I`=(4Xy^uu?a>3x|+m7kT23MOitaz@=;=2;z9Zz96+x z3|pc*B!3??GGPvH909GGGP8Vz0yBb%0kE)nH(0c>IwevoayaE1l%19vK|QFp*p3x# zV`#&b_V4sICYU{&yY$Bvo7iMVq7vq9R&kk;aP4w@dIN$?zE%--u@tu#?f--f>ieRS zZ^a_aelZITtv-Dc>bOvP{wM{7@Co}vIzt+(X_=NfP_(EZZW9y~|FpyJ{MnGZS*1tH z4^eUg*Vw1xB>lVWYx6t4AP-F0u=G7~;fgwxeCrsIqP9QYUA8;ZSY0*9Hhje2h9jM| z5OuxMEZh8Ew&5hEf#B5zZx&rv^{=W{QPf=@95-3Iu;O;D*IGzc?7h=|*md*FRCOIgqt>F1Qs#K{b1 zGzEjWbX*}_ymxyQ>bmGDNP(YxIxI;k$PPgJh9c-Dvow<`dky=tN5V_beVDni-%SaV z2_zTS5QU=Ohp^VbP=@YU?(mXYm`r1$EA(MT6f1A&j=a|F0nq#SW1cyXea-^$G&5!B@jxG>{cv$QMgKy+aGBxVcfPSoF);kW z2?E7}3n|kZb#?UF8RN7pqRI+6{P{gG5sek%ivm}gE#aTEo~tKi@jZwx9)%%pI<6Qr zya~015=m@+M9m(y@Lbx_Lh3_S87fGlAL02us`uBfH-W-x_GsGAa2x){jDu{T1B&*6 zarf_5RaGkbw$g;a?>p?G2VK=4M0y~dLUfmf#o#3ade#i(EZOi!_SwvleWIBYl`$NC zFk8z+r6t$MEU7WFiPb%Bm|+YcY5KFhcs>+XWPJKNcx(RfOD#{{)W;owIqPBO*Koq~ zco*VSqce7h1erZdJmz|{#gq$L{&WBvH^RiDKY4h-QPj@n}#irJ81&V6G*P8n(dV=OhhR2Y0OdIVi^cx)&z&U4bIez zEt%#)Of*R)vZ&c3jnC~*;8?Np%OIgLmYT+$XW7I5XB4d~_O9IU6z!kQ!^JJP-M_SF zG!=DZHB7ym3|W6Q=U;33cp{S!hJ9;)P5aTlX}D2eHHgUe-fFpdx+H1;roPGRJKv}8 z__~Y}{vA3}3#yip(xjGQ%INB*nJUbm^5n7ZQ z4J#Yp5S~k9RLkI=2E?$$jUZB%rAmkyS+ytkt5Pwx-w04KPWt&gU4WSb=7d7<#aJ3B zdrR7wXGS6$ntR}gr){nKHl_g=KDPon`fMup$74y@AeNj~xqU=;Uhez;oS{>YIa_0q zhD~H73Qb{#jt4-`gHl{$Oh^9uf1_w_90g}wF2#GwFJV8g z+DFT2E(v#5cl>@YHSwWmI9Wuck*$^5t>tnGmD$|eiDio&H8EJ&lMD;mR2@C3z-}aO zoOGKM^tvP7mFR+6Nclp)@LExzVUf`#W+mrAE6HA~{ z@*2-bKP!3@^F}Wa9!u#(JMGi)7aPQ`*NQHmW#U6(xPFb~W0>wU#@&ccneBN!2tHqp zk$9u$T2p508~zX!l%%T&8}6!xxo9xpM~acC1HBd=>Z3t$rGQC;4MjEOb9bkMb1lqA z7o6Af)Q_PUC81NZ_rM-hffZfX$Y@eC0g$mL#dAMP1wK*0&~20r&X|x7YHa!dBnE1H z019wMK7D*V%_mIFdJ)}ZI2@fj3*e{(xgP(9j}`@R`9aE;Ccp5CvslCohWNMUC}4&o zL8X$;&5}{^tfpqvKGx6l4)ViO#`|!mXv!AP&Of*Q)1~yRLXXE#9kn2ky=3HT2cXF5 z3!x!Zl%djo373`s^ z{*c^=r8JUSHaXUYJXkv-V`dj10(JC}(lLeEg0_N{b9Ul%BDu+o5E2U@1(DE&xk-I_ zku2S?yl@p2B{Nu}MiBE2RGSVz_92PW>XB&OIjjStp%bhu@r`)>*{EnAu+3s&zZLiF zcLAyP{9*YM6aWt0Rq~#~eHBa$NBz-IcxXB1fxOh2kH7`yq7oRRj;6_C5QLOKPLw6H z^>KA!;Fl$=fo2opHbAiu7*}%c6iGe-4Y4cJ+N`6No!j`k-`lqiz^ zAbiTH+uto4PiPlf3ClkcszyT5U)X_AHJyUQ2@)oHP8=VQEV$`Ue3J*o(Owool6A!I z4~Uc}9#Hml+F9E70}nU z#vpr}@yXcezo(t7bk1}bl5K12`pVl>0f|e>185dP#i_0){ZQiiC-_3y#OF~In(P#T zDL|^W8;zDh%n?JJ&P67s#494{9j~vZy7;2(6P?#v)UA~zs9)(L4WY*2JOc~FU^2Rz z0aSE5>}0kFM3!&G_W=3VjU`yz{4-OupBLn=sGn)m~3P%W3P;4xM5dQF|h~e_$_#-WaosZ5L^%b|Nr7IL{Jmx!(vAB1+m^`tE&L^_s z9th6#o8bF*3N=4f$KIyfAFN{6VJ35!<=oL(fQcMR?%PVs>P}xgYm(?l6O#q!If+va zafcr6E_rZP&- z&t|n9u1s*tX7=}Uqq-Yuv?PUKSdyu21{#T!sZ3H{W?jr9)s)D#H4G`S!34>QZSh2JUkU&Fpza1J$en-NCSa8;{Z- zZ|m~zj$9Y$LKI1Q0|MMiGnvGp-s=6Gu3sP(Xmhll%ZlT=Cc9# zN|8>rMVI7(H|RHs|145uoa#s#I|=Kqo?}4o2PjtY`K!3Z3>B3l2Bl?I5l5)EK!-AJ zBin8w3y|jhN~?FS%6V>;I?()1Mb}vnL2q*tX?;DZNz~+BZ#No$KuX@)8cEh~%DJIN zn5L`p!7&*yFNyE7llQf_R+WQ*T0itDLMljsWBWSFh{0oa_K4RbTVkL8lrGTs6wWsBp{h>LMd*v zZ$-s|Fsp@C41}1EWJkTut-42%yLFxTRcKAvQ6x4VG~`dXzf!7ggYb~22+1PFit*Ak zEQ>t8lBpTM8Gm$s-NRte+g+3V0?EBO;h8saCGG6A=I9jYj3IJNh%6)7>|W|NgOqa# zF_e&x{t-%i?mJymg4&MJG;^?Amhhh@qhS?;K_JfcS6o8K;4+YWlL)n}=$hON$6B!f76$ua*WS)qT1 z0R}s?le!WIFVM?$EBBET(A{cU$Th+3QMCn8;9XTR9;R~M6I)T&?)=H!uJ{eXm;A8} z+>o1J#kHLv{jAz&8Xm%uDGvLKfJ*E{!Oq2J^Iy91g67lRT4d~Bi&Wt?4)H4X74hf` zVMm)tbOP9B7;sJQF1>sG&7W06kVGvIBEBK$%EtRxO^8Z>M#7By3i_ogzWND{V6s+x z#bv|dkiHC-8xqrH8N_GXtIcLL8bQHr<-^asGr+n9*@JZ!t-9)-lcS)=eMbC^yP~?# zd;Cle@;kHTfcb*v1xZA-ig$U1mGPH~I1ET%>5U*9eauQ`QV=1EelxLA(>~GW7am)( zKyT|Gk>!US)`JS&kTz!{rfBQ_&H(XTVLIEf1aDp=a2|FAy$tarsRrW%bPuwDx=W>FXre<7T3QonV4RhRWJYj^X~}pes_G59DkkSRip|r(?B4*!(ryxduiF^{SJpH1 z><olR2=P&znqi>8{# zJ`*gUnZSSLBEGIZLvcfxtF<1?G9?b$Y+Tz&gd%dmW6SApTU2?z_Odt-ot=n*b90|k#*kVmEy!t zEqfnxK%^lM4sPiQVz*hm5Mx}tJTtCZ)7k!TACuqJm?VHQ_u)7=l?UdbhOj7Mwg=dDcs zydo3y?Q$cXM}(OfYaW#SbsdLH`#OLv^UHmy*8S{(Gxy*gP+dr zE!VB(&;-Xj#HzD7QPzkux7;}()A$qyiJ$!Y9 zf3Hz^`O*Ed^`2eR5b{qY`!8Jw#mtwZQRS;5O7Qzpg|L#3a4^qIZ1vY+up_l*Z^V_0Z@Mw~-~nf_*CH%pkS2C>*# zj7i}ACK~&^jWLG+U-t|B_nz-9Md|Ln2{n;|e3*EMnDDGI(Bao|{A=<_jjJZbW2ztu zA$AxdhjcvX7Y!?!uVo1Q)iponpXzt1aSGD|UUQ8zk0c79MRnj4r`nvev!k*mhK&Ohvz#Ul`|4@}rjRun~`Iqqu(Otu`>x0I>Rg0%G8A^Uw53%#X!(+~79 zuA1gdjffgRUkmDHAq9C;FkUD4j14Go@-{b?;fP&jcB*9z2Z((Um2v9VJ@iT%ZvD1v zQUxuJwa;|>%yQ{|zFBMtmm$iv2)dDFg>#x`GAB)0$dZua6Pby8vJvxh2qjkt&|2Eq z^V4U%)-Ex;m5CC2U)7bBeNp!K`WtSv_cpZF(PLmhU4&!{yy-@rmPRP#cK8vP{t`c4EpRoB{VL~(vF_p~)I3;IPBT}Z& zb+t+D5@~my`@EGWHwAf1-K?-h+N+yTj7xFOLvw&36q?9DG%P5TtAamciFU3o4$}kk zmt2yd0#u?!S2?qCx)$;RJ{9K%vN+4qaeX?9U(C11Sy2U8qnwm51Q~{gO z`QTJuzS*29Hd?%311CvufX=imR6 z*5F_XZ0uRy40F;iq=dOd-|2LAFNTOCK3AD&-wwV{x@BJ4v`F}=w0DBdAdq%z0@?_j zUe&*{qTZi;oon5&oFQP-M z$%}3rxv5$4zo=)JlUB8a^XK4lfBvu!o3RvaKak|e z#VpDSY`4T9=|$s9C1-0lo{ZpPEkF}1C4U4*lAk5ZrCuz62kTS>%A{F?xwop&&va=U zmI|21M$M|*YWt1fro3mZHG7rqO~!&Evo=0TN?x_)6>m$lbF}xZEPj~&>u%HD!K>ZC z*53KVN{_Kh+QDYXnYGrr$!v9cdGF}%m+pMuNP}iP@^LDRQ6(p_HApby26UD^oV?{` zo0oX^Kude$!L-@cmKR;4@##j@DrNCx~0(Z}sj}FW-Pla?3{Y zLFhKon%2<!{Kzj)D8aV?!i`z*U zRl>ap2uR=9^-`ZA3wC@M3BJ)|t^|3j|0KGnZdXN`^ORq+;8fwmzPJn5LEeZHGEz9O}j@ zDLArJcc91LbUfzc0*A`Q+mCQgBHmBnYtE+0x23~D{tXly6$&(`r2u|A( zm~vChK*ZZaC5Z*VM1)Ioe4V30eEG4rc^S^;#?@Ww`THVn5aKAv)4a2HBnuB}{OI0} zhG3%wCI3W!p%sg&w`tPTuNAe(lEp*j3|uXlmI42e62xKc_z>P=s=Up;Ya7%~2g-pd2H7;&`kqd7`vF0Jd zA(07c%=c9tP}Ok?d$l?xk-K*|eAvQD^Dyzc*+%dBk^#g`z(>zR0|}Ijv5zx+w^Cem z__0f(Ue1^Ahry7u4*#&j-seJ-`Xrs*FFE=TbJ|BzI9Q2*XCKz!=z z#YCWgJNe=FuwnL+Od-HIyplnF(H!bntv34mb}jgQ!!snG&}ef|AT(a3lVU^Wgrxv4 zcGAQLP;om!Wk8yp7C|RfaeZKkpnW=2``D(&3AbxFazXvp-(PieB3|l48nrLn1H@qj z?qt+5p=s4RSu%%%Q}XD}Z5jQ#!4E{^c5EfaL)S#&!DWS!h6pa{f>81J7E`y|WARJr z64LW8WlhQmZJ!sjnTNQ>sp*!z=1YhNcwhbhie5aHuvAL!ru0vkT!MZbJgpx)TLe1W z&=%V}U-s8lsbzcr1ihygD+-Zo3}|u}Aq}sQ=X2*5He71Hnc7ZmKr4=xvR#1)DSJZE zPHnT8i8sB-iEts=5{EnO3xljK@&XBoN6qLI}`-pO%(;>#@keGT)RS7N2{ZhHFeKq_$wr&e^ZDt~Rc&OS8jhi|* z>T_+Ket4$^6!%Xs@v7Mm#nu*EezA!j&@UaYo6-F-6Z3Gb=0pAT`p|AWqQmP!<^3EG zbSyThQ?NF-*R`^cV{wPXPH1;t>Y1s@LsQ92eu`zP>Jf{$H~zgrOtC}vsM%~q2kVMw zBmw=omuGxfyTy%l87oZ@lZtntj;|SC(Drg`HIqw1E0asjNQY8+SI5-4kWrHbC@OVX zsJsg`ECnspRr%8bDd5XblA}AV@?(-r`X7QLRC3N#J$26Tvj*YJ^IpIim)|BjALRLL zC)=NkAZ!~UZtmXrc7j&Q*IMjqJO%I9N&Z8xF z?<#m?kLn2nnl~6>e{HcKXnNMjX3k)l zB(R4G9C(s0=iS96|LKu#b7d6I=w|${qyn&9!6^PpQS!1J`?MPQr)W3!>&AEqCnN^> z+|&bKw~g2t$oHxr(Z)e)JNErzv`ZV-9MucM*fyE-eLC@=+)6?ID|Qu}>kUqa&IGxm z3fftWh(x83)Q#^lMz#`QQz&tW@%wwP!I|{d(AqiHu1XDz z%FXY-yZ*)BwT!{GpyEez~*(b;qDny2sMJ-=FyEeZP4+K8a*@ zk@q{etYb+?!T+gft=<$nddIvQjxQevD(r5Y#b?3N-WkhJBE=NR@TNtyer7E_Q}_l> zLRE|BOjFDmNS6QMgRw_KE;Wrkv-zFFzy(b_|NPJSIWba@Q>XP^Z!i3b7qG% zu|$AMsOW)5XoEv=Xc_I|0b+Fy|SM>aXeEe9m$J$PE@GnO2Knw%8R{W|8G`b)y zu{u$eNI8^_3&gui54>MO?8Xr$n0;n94G)R;K^aD9B?z5aqbW&i&atk7DI>RcM=FaF zj-DN(0I>%KXN4P7K(zd7;Z1_r9k4#gDrBInMbGr`I`~x4##m$6?LM|FH6`Bon~b16 z^y76MW|RT))W9tVqCDfhkNfH@2we69r?MyMFif(0p`0$r@5QKpL0Mq`~E*j0w04gHZShR-tXV-`!cgbvZA!Xzglq%q`XA3qzS~c zD@897%6E+%@16Q_)t*`ctk{aasI+bc#C95E1Ri;yepqh&{gkd5ml;H| zILJV-n%AA)ug6xV7$UT5d>yxq3KcH+r=tW}lP&UhFvf z*ms~e#YBJMvFESvjxy$-&UY?2CIWB_=}>&3zNc4WtY8%1V$nvy+2=$9;R|Y&6~1S; z)eovkm*3X9>Sr_TVJ*4cWg|I8K{v^b_XXJn`(v99<_}Ybq!Jf=U^fG(Td6v0Q-9fWB403kH@wV3N<;O*g+mTvpV{04)!#H z_>~#p-uD-KPO+fMPlR$*!!?Rl6IcsC6gJS=)>v+GGD_Ec$ zX=&xjsk2!+02Q8UGLU%io>Vyuj~C*Ybum0YSd6MBq^L39$ju}U20=~aYr7qH`>|Ar z8L$O2sxqz>^a; zNrwqUETR`tjpO33cQ0xi2xGQF=g20sb{pBJ>vv~8{7~w@+7~r|5RRvR9+lPi3AR=~ z1Z(8>vyRaF3}l*kpzSj@>@_><9iHAT;`@e}O9T)BsNU;zGhgB7<}c+>`Q|-;;(q`6 zWjz{*DAV=EcA`~^#*xg&={ff4aG8|JV5rj;6%*yKi8VXrmWK*ET3*mljI z`yHF5JV2YU8$L1>>NkC;lVm0mvUGXTJNy}KMHb|08U>v!q=Fl~S?~hpAuWlfT=$d^ zpMM5A^SMc%`UvV=5L_K_@!uwN|-XqGUV>{{hVwU$bPc-{cRGJpVZ;k!XHj)spU4 zWCcXf2Fo726Hl7?cC#Jk$6*F&1$f%wdd*0cU?Q_064%jxsASXxnYqz5CHfA3jh!T zY6ttXfsn6wt<_)~#2Y~~()D>z11iJ6QD^c%X{a}fPX*Zg&lNYSK#n(!8<{kKUT+wfkj|tT_7pxUr84?0qZ`H z0Kf(c0D$&iOeW*tcwHdz8yCT#oUY(2*V+H!LYf9ozH+^(d9M4arso_OS?@mpquyV= zm%y_B0=BOJumU`-2gFKLT0uZnk_AFwLH@sf0RDR(!fX{hqX(pY6Z!lXAY4nVTz&U{ zn1%IUgM52Se$IR5vdVeo-Iko4_4k^zw5 zHI@Jf9wh)$faMK8u+8pe{N>@3IHp?-bO(3 iH-?U delta 45842 zcmXVWV{jnf*L0GNZCe{V*;t!wvaxO3)`T0|wr$(CHnwfQ`};rdr?ge-AWPEq<=>#t z0a(YpEW%_f`Tk%>m` z;9ktf{5d!4OF}io!=hPN6DCSzerr;hIn;yk4oZZT7M?C;S%|6y{FNVgsbY8TyX z@AKDtmH4P2`Mx1VZW28X>2l{Z06`jUE#YILFH0cD@|TtY@9gib7Vz%d$(M&I`_Vr8 zNbqTJ+tamQ&spWpB4ryJ_l-*%*8bw1D(xy~%8IC^4#nn`&vlP!k-s`UA5r>|PWZ)s z=E+(q4r6*`p=6#ya`PyPcEe|P=fFJ9Ku1lPR2fb0325Pc)R5%P zvVOQ4-CEGL8TLzUBO}fuB?XB^je|hOGDF}Iy&%p}Q+Ny8UPHIfhj(eJvE64X4j)la z@x3~|@~b?07-`%)C!1k+OgsiuXmb2`owd7vLH`u`emp4#eCGyeJJ%YH>_o(qFB1u% zWo~$s2=i-PLocaqQ<6lBfe0;!o!hilCjx0e_zJ-yv9tf`MRvQ>>NPwExw`i<1aD1N;F5-MG5jGV@ZbqxeD7+S#T`8+FsHb^dB~GpbomWBNlBrD3{G z=j`#lffZR+&kE%yw*4KX4y;M)3L6z|5@NhIe{`52se6lXG zzL7MHacFcdcC=frUt~wztO*e3yTNB~7zpcAARw`w{@sw?Ytmsu62EyU(g^)|b_mfS z6SmNhpLmRmrvvT@ShX#zvrk~nce~kVzQA(`tcCc3JK_CQMv`*It?C>=e_TuR6mxw#_2l;aEApBJG|fA7i;So2PgE27*izxVDblg+UD5 zSbEXleLmkC6+Rq7+m^t(J;nEaK5(|FX<5~d{4k1&l@1Y#7)M_h8ns(KZ)k9E7?yea z;w$yDS$|2-RwgF%01wNu*%fJfuoi)jr&;Y`kFxFz)*5vKf=0Do;f6E)&I<%k`q;P2 zJL~1kH(Dw=+RX184Gr@zz{Sm1MzT|E{KKs_(&iub;eT?0C19)Bka%?pB50HeH1#^s zc*u?bZlS(I>p|6Oov54@MPFG$-lo&-A|Kf~TeIo?!E18NwAkQri1t@jrR?RADiqXj zG_H~9Hky{e@Jiikm5Sv9_ygXgM!jl$t?~O~0Xw2)I|yz5mq zA7hJ(b+%|liEr+5ij-@nN$2vFl`YFn>*JM1=bF!si;}jDm^lj1*w^a@Gbe3iiG%EV zsX0&5UEIKv@?L3|PO;_E3D_fdB|g0r^RuU>jaCLQ`VQgOI$ds8|_v=*pI*m20 z+Vp5OJH$#%C!g+3F|#$1&F1N5q%%4J*Ur49S_2pF8c1%V<_S>B3i3?%0W205}7-U#a>j5a%m@G$+f4mrvvA{H|MWD zO$ZF&wkf7HJx>q)DcC4UA9M7vbH~@_e3KGtrjvi2EI^f_AO-P6U#&JW@GOf>Uq5hS z+l$Sddi@#qYenVs!KZ$+bCgJ)_2RCz*+$mn5X9~9Z)aB&lpY3a$Jvg3%b=yp0oyeb z{p25vSy7);qeI7^Mwd5u2xwMX1?l{EeU&>v3ZEIg@))Xpp`U7LxrleQ4uzGpfFXavaj(O>yIAhSX_^EU>0nNWsCo6i2$I3WQqJn%*mfK`?sgR%Hl2UrS zGW_nCj*G!8{nZDeFR<5n%wuZJ!%d-tMve@EM0GK5xOkWSV1-n+heK2-_kXwAQKSw4 z7an*^K4_wPDhcRLzD}~Cy$s%cP1i8P28lE9-I|CT&-bVTnVjL!5MbZg%wm|xw!DLp zso=qju$Vo=B}>Coi+{a}G*RIOZ!r__J&`^&9$jHIZxk1|5OI??V-I^HwaQtCc1BdD z2;{0vkBi){nD*&54JPTbhFJ^lC5K)C(;a8!v=*Myhxw8lsXFv}u$UVpm>5irv|AMW z7GctFVYd~oqHMMv`f|8FtGj$Ss{xrmM6~Y`#$jA-11AHCiCw_69QCHuYjjtHugy;7 z7HUt3(ZXt2>XRz5H&}?`h*kCG3wnyY;jQK@RER*L<8ku8%8mYSe%aTQWaQ0+IUeEwTO_N6dVcp2`{;A1VcpZTx4 z$3D+{1|g7vEVKv4S&Jc`ZB|XR$5@e7>!O7Z2e_FVVuG2iWvM9Mk$vlhrdjRx+aVX% z;v`qemv{ZfiJ!baU1GT@m#m5al#`L_R9%$r8q?<~;V{uhWB>HMfncgTSmOiQaX|(M zJaa-t$6-g{wyaw4+^jwvTJEU81C%ZcQkM+W7SU#IuQpO+<1eToO>1o>c5FuI=C6RQ zYu4~Hq38??5_iB4HD7mz{*k$mAyIo*=`Ko|dit;xeA=fyehR+<3>>Lxb+ zAx!hdA*7fCjs^fNsqBf6oP0W#c^z-x-1;%*VAg!;d=Y}gO_u%&mE_-$bth%sFe75Jopo7lBuN}IuwML+fEm}Y2(~5AJG&( za*f*)vlILQ+F{bP7@^y4<_i8)YC3j|&;6A4)7#*D;^iWbRed)PrMU7qY_t=Y7JJBa zz6`+Vbz~qx@*H$Vv!g%F|NUXIK{qyNt2tx*mI^k3#{#u=#rp^?o!i#W2W29HtP7(H zRt7ZCHLyyxLgGUQRN%hFC_P8o{u?Uf`!&~&&XHl%M977wFEV%I(PoT>Y5ykLL{zE{ z7HJQnU$MPS@G9n$Y^Ss7{Tnm4hC+UR;t+5=vb*Pg_)8VHKEE+>^(NTRW-ij@yOO3v zi3_fVpF_h-CMBeHwAK6KR$ zTJR6A{;R2Vq;A}FfzjLLEh=e$Q`OTASYH32PbBA~inoh$G>$=m|>#m&AbgviIP@jz0 zxo#d#Se>XWo;gK~0G%;?5|=X{sT`mXI~5%Z7ws+n`+$fFagMRU;~6xHbHH;|@v|z} zFhI@iN30YosLRuoAI;r8M3zljca}4W3LzoS$zPrO`b!9+Dlormd9^Ccfn-}uyxC4_ z{m=4)G4bW#%5IJ9_oXAA9|+5glZy`Rm1|Bo&8uk`>e}G+ZO|HNqAQH8$$>yslR%iP zZZ}Gt_oC|x$oF4d<6gC)Bm%F>5$QZHJ9;|yHk5PL&|`8D=|3KNEmXYHBOT^aGzaTm z&^}+<4OS!Q@ZcomI?{d&UcH4Wvd+?@f~+3pQiZ8Iet%VzwYTK3frx&K|HqX)Lf64Y zcQhh{aEvvRy!F2!*8d*oKY^3SV1vI>fpC2l>Kvd>A4(HxKPI8HgKAcxgAwM1v@=g2 zklyWdYb`I8x>Y@_mb|BK>*>agKs89mrX@t`sEQ{29+CU zHWGd|i8~;9!W!43*eI&h@2+)>ei@cH{rl1xSCnN9IvW`kX>@5^3}D5Rm%;({_v`Jc zo4ROHgfpq^nk1l)h!B&$I5bCCIXsc83)Uw3-6VLxvPp&3{+xjSNjkn__z-Kooh$6) zhsr2*swP#WE)tl=6L!6B<;bB%WkHJN5Q~Va00XylIdCj4%3ZRHHn zr<-uV=a$cXc5rX~;vXtgs*1mGH=TOpur{X11WrV~^hFFC0;W$N$5FBBRXhqq>wBOQ z4Vl+ry450Zp_gy4CJx27L@Q}fo}xlBE+g)SPWy5<(e5$vP&(fk!4OPYhgoxb${O-6 zHaioE5Ym>#0G&mA5aJ+LORLYo(+sSR)Zh!^$DHGg<76pqLcs@u>7zm;wQ_Zp=xL-a zRt}hZ9RKE8NUWmi@d=NPI+QnVyokq)R!fCrAL8(lwC`1YV`7XKwjZ6)1yw&6E+ikX zE`1VR?n;V|59OzoG!Liju~yjKNUBhlbYH%2q)r{%0NWRS^KK^1mQTH>*bo_qC@RTO z_6bKn;q-4DEJqy~tLj&j2C0mJMTDPakkWXVbzSFYhd000UCXQ$`QCDuk-}4jAT$@= zK5nvf&?v}WYz!8oYw@U9ow`xh;FyA2f+Os;Z9*_-zZ(V&YL?9)W0%2t^(U02%s32?!Yy!mLXpA)t>xM* zX96^t0`c4qz2}Zi%Dw5DF(NkpV(6xv@Ikx`U>>F$Ty!=69{QSYy>`=!(jq68%Fll+ zc)=kwvGg%UU--aT$0*5yfpxgyZUZ&aN>isnKu^P82I{gM9sQ$5+hT<;RNast<}CRy zB%2GCU>e9z9qXh`h*NQY4KR(nif-*hj)Kx6Gx7Nm;?yeyc_-|Uoj5EsaVlzcYK-A> z;KQ7sty>*KR6aH(IRWD~XxZ2c>tJ@Zj&tdjZBduAVyf0G6k_qM!#H;INd%yGtzjj(CMA2!@q|b={4P z51OlxQ08{!37p!h`%GHa$$M_M5+26m3GoXp8ATRh!YDMOOJHLae3TC-$9z3 zA<+lipc1lz1}84%^Zsg6_DYrb0e&{i%*R_4t_AAABuzFcK1k;FYHvFYa;?)*D4OiT7S8?Mmf279=a(Laf>Yyzy1W*E)0W{g zvYm||VHs1RqttMQgfLOpCA=0>V$9LUL+Fe50qPiQV*J`?C$e>%`?$x|epNwGSq^{u z5AFnPDTlaXTfN$KoOu9IqH{SH)}%8NZ5D`j5qA`Ze^m%b00zLR;SA=cbr6ig6dOF!y>2 z8{P*s+g`_p)?caH8qe{LDYom5FeC`rk=mqrPMOSQ!dBAKd)mI8l1_WiXoMktm{lie z(5$Gv!J`wL19{}Aplr3!FoYTIyecDQVz&WI5QNqP=Hs4s5~MXIeUTHutymI?<}(33 zB_;ccznzSk)esqL00`w~3rmE*f!|;Ayqs+rhA%#-$Rt{tP(Y%K*Kl8yzKd*GL}y`@ z#GU0fFN@}uRPlU}DTD_0wYe@T+t5RGSAWhd;#Goe$ONtFZQw_WO)ByfmW#1#GF=UY zb2DshJ2H~)IpZ;#@_I0^K>qh#BPX5_H}dDq_YLg-%xVD)?x3xO2N0DJG+ABE=4#5E zkQMS_SPByX+T%a|OCOgU|FMIg$Ssd-aod7!yCg%nw4;Aq@S9l221+|NjiScgDR9}VQ~VUpzr z<%n#z&W*}t8$LDh<)>-hqZLP zVnd-%C}2|6w|QauF_sJK@dVzfW3}Zul0$WlwwKX+>Fzto?5m=uB~sqFy&&dS3$_;! zU~k7=I>P}6k=jid36Z36m;!E(3K6tKBmTi{bRg&b#OGgpWLGe%p-dg2jO}R_@gP#& zO@03g2Lh0m!BAZ*Z+m7~Uh{-qVo@tihLu6yxMy4j zU1>>4Z+Z2aXKsPn?#ND^aOm$dndsBer6!F--|)~~!DYFO8px#^ z1}Z1UW|DBQ&)|THan_lvZryd4?5wH5cvTAvn^*Q%FY<=^);gvGAb&KEau z%D`Frp247W&)Lz4Nj@^j*qlC>bL*n$;@UYI+F|!KyJrX&WgeW9OzYI$rVSppcRSW7#&>C@A&G**+@ zxaj|G2s}In-4u(q{OziFY8V+(TqzBDb9(z`CHg$^WGfVm0LzG>bBSuZuqFrpd#A8Z zGAKFQ20;x}03unUlkOJ4*X%v$+E&=!=c>*%;u#@*g!{m{g7YC4efuH8LY1P*RNb(< zN1oyTB4yNX4;GBajA~UQiK|@?B8- z6Io6~Wi>=Ab zeC4^F_>fBn?-6CUKQKFF(6!bJ+Fl?{FFJV(^JoMaijs6`^l0X}9`L`yGSzuCot6q;2BsbQ{6f?*k#T|FmR zCZ*JO302%_rwVX0iO5T$>@I9Yh{%Hv?{6yC$iRO>a3L|_neRHO zI9x6%T-}<3jtqNp5N5TqfsR>^RurNvkQtZLXUBP>5RnOYk}@xB>a9D(wRB|edp+y& z{c3<7J6(JT7RZ{(`D!NmI3i(%@u!`ZjYgXOs4*Vt0(l1-#HC>1Fiqq#FEi#Is9WUZ zjFBMPbg!&2yE3eD^Yz?_m?~<1#1mfc9k%{x0dO}3_`$C|qYPAS-?m=+Ke=3&?g@$h z1Pt4jtOZ;Rh{94ve|xRp9!G8fKsk;!n)FI*B7Lg(_u&HGK)oeZciyW^MFnhjIlnqO za_qWFl-+=gH~eE&?myHe)56nc-HnMdF0iNdLVUshSn*}&xpn5imYm}VtCaj3lB=YC z$Tbl0_wWqPmFL$>>ziiw=e};T_LEvs`@st1{fW*JeLi?E&E;@l@1Zq0IH&-*nry8% zrii`Cz@hW+5bFtiG;t`ph91fW0zydVMl^4#;y6IDRV{odcC=0Q&#-l#;qz%|#>sql z7E(U4seP}YgSGfd5@A`>JVBXBX|uiLkekhmB+K_K5aE2_sBcS5sOV@N6}Z}A12RpZ#< zfdz;01lj&Qqse8~?mbln2Kv(NZ?oQ${a$a)$pB2(>BO7iOk{;Ea~5NlxH)PU6Nx(N z+a9hY2v9~DyNw@$gr>=4!0;{E@-6kdodKxg2Br;5757VvfgB&_VjW}%NPxH$5*1w& z%$d2&iu#vli?}fE4Z$H}2Gg?vMz(5aVBT+L9uHmq*sBv`e9@Iqx1f76FpEDiT z(YmL#EtR~B&QR@ZobUzV(P3gK=N>cL3Bw+n3hj7xI(X4G_uV*G_=+IHxuGxN{xQFw zL^OjLzLmm%L!6C^WYLM-6}6?P<90WdoMt(?#(L{he?bVr~x(p{bXQt35bC2`l59pQ6{ zHRsCr-O#2@s`B3LVTirf#xkb4Et9Y@Z;wGG#frG*E9DjAkM8I00K4y9 ziu|MK0D=SKKv`Bu3m4LBa1VTuo1HXS$k1dwtOh2{bzM%}TsAzXl~{xTAQvA>;>5V} z&^lF=WX!>!jhb9dbN)EuG(p@bQYC)Go#>HNYWINe%L7?r#%y2Gg`kIYN>v_T!QN=m zIc2Z=%(NmMd%}7EC5H+PY9`U`cU+)#v@ybB8zL>SluiKkmmu&Iq$^OrL;f#QC(rZ? zTm3|EV!yU&uwO*j)WTjNz`I%*4%7d=?ZL^QOK5Vvz?Jp-aXu2ZdxUCq8I1FX`#8+z zM3pA{jBp+i~@6% zeytG$jD=zOO?jMzLu{!0rH7b=rIt-yy`}QoJ-K~{b&spn){k;%033UApLo9f*D$jiXcOfhl|CBW~d1#EEn ze-nNk9(;s*H%1Eb0Z6XzsCzjIX`jBiW)m@#AOB0axix0)SqRmv`bmij|pE z=4Z$8rn;lY>tUXaLV+dQ zEbus{GaMhe+EJpDe+4s=FeO{4{pq1n-Vo#!VQQ$7hA?VeIa@ly(9F|#0QZ#bG78x$ zVVjtb{w-qT43L#{8C^7RWI{DWZ)D>#dHncv+d5O+IHKr3t@$WZDbDy^3(vl|yWh+y zTk4r_jjDn(IG@J;QUIE&v}g25!-+_)lT@NoO3X!1s`40gT`bzfS0M2__qn0uS!k>Q zY|Q%`#nUcr0bQ9-y6~8{5iWIslDIm)!mm7x1e4Zp^ZhfdV;(Cygj>3q)doUv#6}=R&ENDU>uq=r$I&2t&lqxyu z$YzMpO2D>)S!z%qeNYlCJ(9#=v`aPEc&Fciu0;Pj6vY}WYofdl70ss-u1ReU7cpip zp}frm!N`hyNNSMPKwupO>~~gnsNFRcAtO*^>YCd0Qq>gy;cDeXumZNRe)S!Fk_g9r zls+19Muyn{v=c)RcGd}FWqX~1btM%~lw+C%ZvZ2)0d1gcJ-1Vkj`0!Cg&uSW{bP=F zb0&xMdU`|~Hoe{jMcZ5gU8!^^Z^EepLHm6i2^9D9dgz!qtR3 zaMW1wU!rgEdI)z#8*e64#WRo7NgxB3OBM$8ol~b{Ll!-b#LM@`hQ=pynx3#D02#D0 zNc{LM>vGcETVufU9O3QfJum*czGRx)oy=F4{KK+63S7VM zQbCg;)sZlX+bdMGk77(2@q=EB#yon#rC|RhOW>kwE=nHQl(0gH+p$G-7)?-CYRL2T zo+mIv@W{%gi|&#&i65QTbf0|;B#$LKI+>s`Ij$bnaJBT!RW2&N{%ifTgq+<=T`n0w z+G+?hngT{-vsd7qNE#G2;nn;uH|HU9)+&TGgPI2C87D{;deVZt02wzc7i_KVxPN2R zU+9Kc-kalTKJ82&FdsBAX1=3RYr@=akh^9P)?O=unoSfo7fRG(NTiKXn zO?;E&52iqu5A03t22srQh!(=Z*};L_oNxr#>JbMW2Ki*41(-nttU7BtIGifheBIY; zco#1de}nV8U}_KsaNkL4j10Flir-qfHDNj{L2fmR&Sh~g3`PCy=pd#fyWg4!qUBf= zG9IKfze|aI>Zc2!q;kCi;9R_yNu;$0^OI3NFU;rOLTv}f)vyM4)y?zXf_tr#igC&E zl(5zqf6F3*^Ye({cv57=gqnUjnLFXw~ZGSn{K-A-dOMo6U zK$``p4(4!?^5&^@~}>p}c0MPRnFfhb5=BGVr`(cjsQzb?eS+_7m; z?dUA((?p|ec9kYs&_&K$gqy}C71#cGblTER< zo$}L9zhV~wHEJ9|$nK2NMPwF1$S8r>dT%l!FH}w|MBF0z#pv++osiNH-+20ZJ{V9; zVIjJ9#jT7R%*tqC3K8*R=tx;-a1CIXr`)6(;eznot2HEdeo@AnvTnWjswT_%)}E0- z3kF)dC~@X9uQFOIqu5#Y#GStC(WdrA5n(wOM6HD3ue{8gW3mq^@l`=;vl~yTd`E;i z3}sTKhEFrdc!WcQ_3p1C^tj_@?sM88$h|vDzfHe?%Toe3qHLQ5I2~2Le`KTaK6zn$ z#Y9E`MMUUo1nnXj3L=VKeC}YRk|@SE@x9zs1tSF1O6Nu#-`Cf(Ijde|U3dkEf$wd? z;RFegh$ww8dg@bP!iD%2?8J&Va4!MAu!$3xf{|PFP8T*1Nj_B3mxCL^+HgLAhsCMV{!7L1K8r4)JL6 z%B;JKt*gO^>IZ+x9(KtdIiJxjBu$@^`|^W!v8K#r$(b3{@qV5f7@>V_cM z|FxBS5Nu&2>s768sz-)sAmPrUrXdX5pRNl~p8j!yAonqb^`I;I_RY+-NAtTLoP|R; z(6w{U-e=#(Dqae64;G2$w$gI3vaj)1){k8yr#npE{!ZgR+rys>RTom>wo6mtc|Z55 zqN`h4S?BJJEfj6>2lA%Y+?rLijXym zflctwM8N8LCKq9=$PYxeE4y5SHT*DuzNU=x)1up$;5i*@J_C&Pj-ITnN8TM?cg0LZ zue?3bSja#`Ob^E_454oqZo78<4k`oQ*^X}bj$=m1>Px-i6fq$@S1)G})OMe>$m9k2GBFUBDO$sGgX7SQ9X^8aR9V$s_N0lnY z87&u0G#y%1Y%Rv^_vLzEhPv9|%Lt{V2ruNK54(Ve^s@<5D$Bcvv;wVMjhqkq8MOED zO{e2VA8Q&5YX|Lg&=N5m%s+r%sYVK~d}MRXH@;*IrGd$ofD|`HxWp}CzIf_IM5&FR zkT6JJcaK-2PGJF_>WEg!Y3fm>g?(S!33Lm(58k>Ctsr$5#2GS{@})vAnQLQD>fEOi zg+u1@$oR;!$W_eM_Km&Ya*_E>1D{;YsY-l5-zy}RyamYE@PWQF5@UDlR|!DAH=a{%5jPcE*l$b=k@i-B<6zM zGy;pdrSuighux3hlmvj8FS|v*wr<*)bVKj^oFz4(lq`X?Vo!NPS{!4EDNRYfUkW+r{wNz$tVWJPmXzzl+&<98rjAsElT;Q&{ng7B zZ)1_mzueoPRLI%#;;OfdTDr4(DOylC*3A8d z_hiuZp-fx;H7Xt%fwN-rFSykX42V2}w#DhOXYaYWJLF7afBW+y%4_uJ< zES5d=2~+BeV!f_)%Qj8kO5xYYRQp^QIVr#;L->nb9bKYZwWrz_fOxjmw=CIUs&4INBG`*q)vX1j)1%6UI)CbxLWh{^lwQCEo*d_J)r?Zc9@j8kGr`Lz$En&0wvwc(T(zU(S znRmPubo6j;5?5!>n|6Ar!r=;2iij%MJ_wk|T(&Z|U5hs6mtzf-LG-2@WAs z!Od)a6EQ1r*ZtW6-y2+gt!QB`2xNdtX(MwT4ntP>%TK1<55s;}X#>xg&jf~A0*@Fu z`^zq|eta-E6!oH`eynk+r=#NG;G@qXNG1C;BE>MzGY`Dze&NElfRz#s5yoVZGS7PY z&h;3|q2>bWYmLq&ohY*4CQPQsMPs?uKg_9iGg_{TJp9WLe@T`zt&JNw4kQ5=Q@t|G zL+H2-p^nm^SFO=Yh0NXUBt7@3U03)$?iZi*)U=W@5odFJSp|1zjc|`Irt_~Qq8!~a zI)|Lu685fh$(#&K3iY`mFK*JRiR@U-q^aH-I%F2-B|N5LfjFEalx)YwQ- zBlAcT%v0(cbc9ibJd5l+ngQU~{tH9zytuf^1kzPRX=*B9N=ea0F0bOmR%J-NQy7?x zsv{=ON6~SMsso0DBJ&GvSR}b5BhP3V)JS#U$|c;pq97?wBPcdfsA6fM?19_CDf>Aq zPB#?{!L$DpnGXr-38|Y}Ju%3C4FPZ?yq$5BA_S#4@!3QwN!Gqw_&qiXwg;A&m!*qTS;ljTAoM8F zPoM;r`2DL(52L~hM%;tF(tSUVgfDdrr1GHgGs+I05=M=jK#kUf# z774{Q3~I7eWy&@ruM_qRdd#+!e{vSC@)c0GmV7S4I3)skcmiI7MJG-uut<*5QPlH)Q~vRb&@t3lBZ;LEk-VWp zTyJh>T~~at_eA6MG)_Xt-hI(#i57#R*5&$>x;J4pB`?G^N(#?CaEz*-N6bpWjG6fX z28lCjo6NsvQ4#>Rk3=r!&^nHn=nt)Go+Y(WSBJ>U?=Eb zkp9w!VWpUWy(CrK`BJ=<8m<31R$p4$E)+I&Radm$uMPX zwVJ+}e3|UEgC|i5lkTuzTWdVQylx&NBPIsIg87d-2Y^VEb z)Fe}j=XfCAW)#T{x%rTEj94tgo(c9@w5ccDu7@1q>+qzbt=$(}2jC_6&CZJ#DS&h~ zq{@3YYVEW6lc|DKn%<@CWMnb@*&kmqPev@RgiJkAFO5bqm_ebKnvqgFQP;+qJel32 z{k(1nP8rapza3rBH`LXJ#GoOF7lWEyC@q7au`EKg`j4OE3;O?~CyO)N;yCeg5VuHC zlu|Kg3x>xrvQ5^dkL|f66ep+9I1wxf+B8IrH5AT2j!Ga*UacohB^8sQ5f+OIU0aU4 zGaL*E&aW6$1kEvz*yZ8BIK2gU-aDNu-~!lr@>D;s*5ABoMqArBJV)%6cNIgYr<+T7 z4q9kd-COWgA?_5ug%Yve;~U!g%>AnnejQ*hnL<7_Yg2M^(>Yl_DVjbYcM`dbVl|Do z!N&ReMaOJG4+lC6-i2#aJ+jA-`hiOT`WjCF1wli7^skV4lQ(99T9$*OC z`OXCv@TT4~_9GVDXW6kaN@W+s8D&b`D2x8bQDg?p5KbwZ}P2U8xR|JK#J z;PrZf#!Y)HV1a$u#-Jw=vQJ?D%gahbB8-YUuRxqw^bdM^0`jPlk=z&%R0K9Ji*cKb z@$$56CIkaU0nNXaN3%S?S|5#FPVb?rB>Qkm+w`0}UYKyCtysY_@6G2Z$I;hyE z!ad|4hY>dakP_CWOiQc0LB4=sQfkpQ%WlyiGMngj)0Eb+pb3b2sKfn`++EEaa=_tR;p9{v*KH7_%pZ4@B1cmyZY<$ZE{Y9}{ujl8(#}(*=xRr8oMHSoR{es( za<;t?0x)*Jdsf&N-^XU4EfgGqH?kdpFFxa#oCf&6U4#Etijj7dky|^>AYNjn^w7{l z9`_0;Llycv{hf!dq^{q85?#dcJq3EcAh>oeIIsrh_)QU{2bYQ*@~2qo~gc!J_}k zx-02FSvSuRR}lKg>b5oY-ccTv;9Vo%;es}i?}vCdyT-q7`Pt=svn^?I`@DnR{u6i? zw;T~)wnV!yguH>C-oc*#;*q~q%`3yO9|AnD$stzoDwZ1MtG}Wy(YDr-iGYiSk>?e> z$KYdk&KBTVI<&ZA*Zj|aZFKT63hI?w7tzPu0!$h=WUW7PaSD2VU+`bn6+x3Q4*Kh( z?W1jH7oQYfsOn;FsOq7jlvjLwsUyN_z12lkFHg&<-ZRb20Lj~fnx`;3;Ul}49pdTS zw4p%M^LT<#%X4ur*4lvIKpB~Ro5+QWvd5#g`8b0r&0IyB%FjpS*YnqZ`=MaEcSSW> zDD(FKc%M-805w)A1kF(_=YP^j!HWvTqzvZ>F1Aa&xghLaMP^?rV{gZL{Tw|&5!TaN zs^N2#BK0fz$V`N?tB&|L5)2&COkCJGF9k&=SG zrj(U{#}|aJa3WSB6$}NT084;gX@?GRy;sx&Kx&?HK&mU_tqxew$4qIWs^q%X!(lPp z0=#pzcv5%xdN4zygB6~ujUtBXh`Hbl?MH)7+#Poz7W)HUH}%K)UDAb|UfsTNxeV9{ z7G0rOR*;VYpyaT+eiQNy-x=kjMs282Rv%GC%SE=Txldv*i=g|=AYj=+hAQRg$xgrD-3DK%kaseQ_`(7KRjivB$8x1JLiQs;LOD1vsQvDrA7DrA&x~5~`lsK}+{%@h9bNIL~J78Fx z-A{{_SwUK+PO6R&f6z>FiI? z{}li!XU`@Fr_={RGkd62DZL6-0wT58hc5(z5%Qx^E}Pa@qD&Eg{gmeugeH{K&aNzJ z3ybm}l3dEMFlF-^3z+~|iP7#}!$@=PvgLNk2;Y_Fj(1k966_C<=H&$Yn6^;{absw+89eBQv+ z?UxQ;?5J2WRD`$rV)Ahnr1R&y%{)3G&muwCM4%ZigxkDOBuiqTWOd80ZNJYJj=Wxx z;M$wUX z#E`VJV=N?{fnA{2l`l~)H;&@gjsKrJKllGXckW{R&z(=sH8oCU~@+y++x%pUwIDxQYI}pg8?2=}KNsXXtSp*<#fRAM7^b;#S1T@?p7&G53gx z90=OFb5qp6&5DjdjkYO8n1Z5IbK~`XNfOWVQ&)?(u>NOO{~kAa)eOwxG&^7&@V@Vw z%HJ-LeY5gr|F#Es^g2$ZzGCr4y!9rDSzMzJV?MYrW#V!!?84ks{G{Yvb9s(V_WN#c z;f`jdK4O=tz8@Q_LY_VGwq&%~%q~^NYb#%zN_oMN$8Ln!8g*Z$Qa;LZW|?T>$rXW* zZ_DIl{F1P811VL2S&|Ud90h2kX(dzEZz0qu(M!F2*hsbg>q3ouexyp++gY~YlbEdD z8vp-2H3s8XR6D2!S7<1wm+G)Te-XVM z?|VM;Hqfa@y`C!Jbx6$LYFKlG?a7*lB%??nlnB0|?2qWO24G8su#k+k|DiP!0I553 zyQCIpq`UYfu&*dy@!#JuLG&iv1F6V4aJJN;**f!=zV z11ibr+J6>svbnQxgle|-OO0dKa9*MA=O`itPvAMWf|)`dSv-y)^>+t`+?I(9F(&V% zb#Y2w75_Dn8DNaE?w{65r>)imf{mhbrtA_p^5EGbuD>m5oZKyqE< zw_SVJR?iKpgG9EBSJXL=5$55+ZipukzUQw`k$51c0JyW2uVVyRWXrvOqWA)331?^-iUVj4(TS-AP~LW4O3d|2Gt zY3aQm1AIcz6)hL2d}uH7Gik{(Ucll11F}F(zg=Hlz0eIaA6@Fs^lCw+v}aLRx6Bc# zU5zYfbT!4MePuYhcV$ATgEPCF8vjGU%LT{YZDzV29F8$OwskGRBIy)Wj4Y^_nkdDP zXeiZ?HIu98n3tWGs7O0d#vTQ4>wJBkgEP$z6 znDR%-GV!Js>T^ZON_C2ZyW9>Q=CebTtV{W*QuQsHr0 z6jWidd4|a*Jno`WKRYieBwkht(tJ8IDru^u>E~P2oyxo@Q5x^e$O5Utg2iz`aL&$N zSNb=~4k&+>fH>w>j!I4_Iq~@xbziLDNGSx4P z#9y#tu(>CME19s*gm5Jcvh`t9!l2ZPx8S`+nZ0zvAIcMylfBua7Jn9a}LLp5phd8F&vW`zFP@8mHeH#kwk0tgC+? zE7sYxYDXu=++C^1B~_=x38c1AX5sT8sJUP3KAi_WZbk7r9;`O^c&ys=8a)$#K08Ox z^HEr6;2hoChN)xzZ&6R`91WNy#)?M_2vt*{AY~uxljU1QF~8Z|ojD#k=f<5+W^6PB zZuLwfxIWi)YEO;l7IJ396)%Ht=C*(AyRdj69@&mN#TJ;JyyWGkHn*E8My*l$|GEXf zR@4dt!Mz#L&9NXrtXiSRs{c`nO4ojkJCQ$KUbJ#u1vb_LfC7QX{J}01RPoN~Kpp*+ zS0p5^>$p1%r?bs)@-%EU>HgDF?7z!uh8;L?taO`CTiJX^R}!vuQN2cN!U2B>%;fl^ zO!kP$-mKLmCVQ>gZb&ZI+tpgGmOP9`8H_pBs-oK?gBr-#L9I)E>+L zb6I#!gW*U;$X*Rp8T6f&r3JeBzQF0UD2HtxP{(z6WJ6?y?*!>K`$Cq}qNxXpyC*jo znN=BDitLljTpR?I7orB{&)9!&#l1^&FIiB(7o0eDc{!i3%6u=HcZI!IZac|Kz$>8I zbJ!PeD4@V93-v@x2FQ8#-vd((vI6aFxi$)`zKT*`9IP0YMDYr$(?gqzbtseqe13PJ z5k)e6GxwxTg*9KFqtPM?aIyl9LB^(S2K6r-JI=@ob+XWv=YUW78FYW3Z;CD)9t9n) zKmoXTzBBV!cNlgL^J|5u13oR}al9JI~-M=_dRwrq2MH=vBjik*j`FQtUJpX zXR0o)faELhqLr{}Txb3=iS_S_7*tV{OMdIRhCF4MZ(n=!g1UenOw_gs>((A*`i2A=G4~|KNo1dvMh2a6@ctIg7+&qz8t< zInYrb7;4w(&xxr|!dPeF^jWZr;qjq_aTevZhkCdQ)~;tK=IV%m%qi zZ*@u%1J)f*NuD!|66~U1qu|ZDpXaddh~=}2A?S~&DIa&_P*Bz1W*{7 zg?`Y&LP2I0$~kkNV`kXUL!o9-w2wyND%{LY!N-T;;PD&k_7s3-C1gOuK>s{nRG|o=I{?p??Iark%t3dM< z2>(;*uzR7P(qZ?m!!Au)5KIw`Hb94{R;yrFc@K_@_M*6e0^Ll+>woKaGaa)ipjB#W zX37zhVx)@<(YMIi>FXfQj~D`j@?(oS=fjeVWPD^ z2@6hM{+EAbD)B+@kTq3@^;pT8s`F*oG?mt6EjF&;?#)?KtHG&j(xmFa=u^$LmTImI z>0BGB=9;OBwcpna<=YrvK7g+tRxG~6N5l&Og`~|6wLST zdFFqM&97U|f_OgFzloUPa;M}vyV}mP8_msNRipbCStgqgwI@s}r(^weJcbTb?FD#% zWJ(kr)EPGs!d8(S<=;3W+9R7AMM^MG;{A}xQZ5NV|oBAL`0NhR_$@}#xB)R(dstEhfV zoW&FPddb3NA28P$)uv3|#u)Y!mtDs*Z4e^&;fB~^;^#*+Nf+CnevYJuDFAVG=?|_7 zZNtW(XdKUE>4M$~E#RA(NZ|ZHVxv$^g=E->*^#t4WQt1U*U&X2EOf;c7~5IoDnftS zQW4TlCqn8OM92?mJBu_zq`kEeSt>28qmU?H$n3=<;>)Z;#3nD82@!s5K~-6u8b(yB z4J_!0u>&U<WqEg(paX zMibf^+*E^Hf0JKWW9*7uvsd z1*<`PY+75)?;5z94n-&ZpA$+OpBQOC={$7p^r0Kj>Q>vluXs=*bX%H1{1!11nol?c z7OE(GDuIK@^Ia-(Gd5o8@|iH#gBw(^lYxhrJbk&x%J$v|&;!!~!yGy6a3FtZ&y=g? z(2;5mophQ*BZKD9UF~$0Ms4Vn!j9rUq zu;N*P^URL9J%T5~UIpPtN9cbnY)}& ziEPz43~*|j5kI296t>c!g{E-Pd*pxmWOZ<^^`SjNq@C22;BQAq76754j z8Gv&|s#eXIlNOj%@{psF^I05?=V(Njet;@V`9)u98t~sa$+H<#aR4e$o~>Qwg?m%(Y=gj?5qifc}unsLhqU_qLGIMv9G8jTGy-AFwvowo^R(0js3e zN%EXE2$c>`S}AjInZ19)2B1@emy3Tz8IQ^~pQk(EU&>Fz zphNY|sW_VBXPP#zvrQ>i@>5wR#>L#Cn`Eg&mHgn9(qG^}tc?iV^GcXbieq2FgY%3Fh<03eh(+JYFLNtakFD}T_ zbBWrvdSVp_)le3*LoVr|*E*C&UH(QMnZuvsp{Ys1o|?;SF6w&}rSDIP3r-xo&J{ zp0gM7TY^UNe1t({42t?anSLkSmwX727H;zk;V7gshF5=%iiNlN+^+;qgH1~YJj_`K zhQ~cp_+8jSmW5tz2v-i~L!T7X>G8sKjVXcmksEZXlfy~& z+-2*CXHcf$>pIbn8RV zk9XWLHPWeFqX9da_ux>8i^UI|P15y|MH}hWW51Co%!9 zJL546m=r}%qxi1aen)6ulh>iRKM%Bt$K|g%#!@Vu@w7ai3h^O3a|h6R@}w>Jv%yWl znrpaHl|z$$&Eu9F#@S;l`4 z;&?#X(rG@GMi;Z_Y<3*68rB$8rj9&xnf{UMiHNj5>BA#I z76rutW~E{RO|WUP(+>tWNig!w@Nh7e%(0BJ3DZsN=plTr@&D2yRgF|ds!siLfV@OSUNnFHeNrJtTguV90aWnF8VOQ>Xx8Ea1j4Z>fq=zKHgcMR`l}jvJ}+ zjvIwWd9NlOe?;Y2qPJ|M<&16FQ$Y9eOn8S-?%{fbW|4W|KjBH=IqspRG_kXpDzURs zY{Xr2HOW)$qK$QaHs}DrVRL_*MKwbduI1P-9#8V;an`&R5EWfOIc&L8N76Mzr+moBb9ALX@szy zQBWJ9l;*r}DC46_pxvQbqwv{CBI_X0a3NeXybQw&Ro~(j^DogOsK*Nvl;3E=O|YY> z$S}k7z+B6kG^Hnl%~U6Y%_6-Y&Fvd~2!nyqaCzDhp)*qzU15KKEX$|mNp9&0NHf(5 zNV8x=dG#3N1A?zoGT0{}i2Y+u=3)A72!7zFm|IrDpJ}*(KeGTYU{iWP(@b?h(=5`g zTRp5fj9K>x`OubWzjBLi1Lli_51#!YE0HTmD7r(rgQ)t8SK%CVuAXD-{`o>&61Jbk0W~#HxRtDHH z)-}gngEuJ;FSo*4>4{4-)rm_hZ{NW7hb&QwM!W+S@xy?-QcdT3&%}t1wv$+S)r%NT%5nJ|{O>;}T{)qcK zb@kV6Bt6~mXTCcpXAtuj6dg++!Cc)TgOk%j&2FH9BPyAQtAnvEtgAp!bk>;)8MLwwExgxwlF5Aq)gfAe&Yz zVwcn37@sAc&8$FlVp00@Pf_y;4yV1KVA0%SI(~l;78|8YP-7{O{cB-4T8EhGb##Y< zH34DTd8%;IQ|DHyQ|F8ghqHN&THDu@{&+3#kI5I1O*?>qKdv4mqx!;V+I1XxeSE~( zP8KX%?XIu-zd=<$rUrSZ(&;crlF#NeJFeh9Ah6`3?upC(pG3f!VRrY9Qe+W zsFG_+V6KX5*7C;@EP?u|LTSxf+Vljp ziAXJEs+*pTvE{8%ENo}JraO*R8;Es$T48^WbH=N1xU-(ib|{v-!a6p4RlsD4e9M@f zwk(>8UkRE+bVyJ%Q!>HT!L#!So=r!2{hhVKik?7l8^9*S@Inq<1~les6i9;#Td9Hy zvo+Moe3;x?4Kdu#EJ!xFI;I`GYZ&%<(EamxCY`Yi>+wu*`|_)a{9J}uAlH)agf)Mj zoG{zh2N;XGiCFIf5f;z42r;Bszrm?-a(ild8pBc$=C(H@I%wy{xnxPQ2`))>h7r9W$f60MdKhW%S7UiTgrI)fj;o^sHj=IkK}U|hpx+uLvNVBbo#0`w5koQ zYC~TwGL+Q%v`eLGLrYd@bx}S=rL=$Tl1}%vbxw=Msy4K$4gHx3u4+Rcj?q<}TvaFc zbCZ45O}OkHH@BCK@oZG6xUFLnQ?S&d?n2Ght?gj9+I)as!IccRTbCRv5X|}Y{)@sRqY1zXUI`)>38P^_Tauxw^0xN%nvgV9KQLkiDG|QNd<~H+}fiQ@q z00Sol8NeHkTUm7F@o?Hs>E@9tC8gt~*r|l*uI*Df;rXaq(dtWuPqcq{PilP}Rp=## zGGWkc6lQq-6hp8;fxuxu@mMGrl?GnoFge_<+^kwrm-0*$Z%HeBwNh31YG+V17|8}K zDl?6Ok+QcQX0aS6*UEo#EosS`R;rRU?fkRHQx@wlFnb((9u+yv(IQeGW2xT5mAX-| zuCL@h5c^abStNTg(_*U8t>(!O`cIF8;jxA=Wy?|xqLhv@U!O!dzV`h6;S@*A0 z^p~Y0jPVN?n{#I)JhW$)bGm^6dS~NIZ0s#d1=b=h7<5Rw@u3D@HZmfVPMQ~a4!Sf{zDu)6PD&$3L5l_P&(m*@~E=FzVzI!+A*KY636 z;s1wU99t2_K$8H=x3|rUmB~YPnu~|5`<}u3Wlpk>3(shav6jel(N|g|@>JAD{zDOi zvh_h;`Y~s$1C@o5q57J~&@bMAob-KTX|cPi$TjD2(|99VTV|I@_nlVfYD*hh6<<@R z*k<^vn@N9S*Ud#@*DEkppEl!3?B-;#+r&O~Gf7{%xkz7nTaJHqH|KdtN;DAwgUN9pP2Q`h0-bagZJaQz zk`j=o&92Q|OsMI5kyf}?GQz#0OeBjugDQdQ5p5TTb<~cS&-daS&+SVJf-R`XGDJ7!H0@Mm<*i#3Uikrs1}#4q@zj71JKDLS~8Jb^|F6u zlIX|{QQgAl{`rn}GBZrOxiUz1qotb6Q&6Q=+Y;5~nXXDHg zrlpyPWQloaj4(dZ+5S0BX8K4sSNe#Q>#GiO(sM13oOAY7`+%-jH-oQ({vsKnDdH*5 zQB5Gmz6l@2U7@iaNDd`g0v{?9xEO!9HtT+>T0U8&H5BOZ-v>gh;J1dQ18)bkZ1awS z0ShC4Qj`vPbuyQ#Wi+lGRt9=dPbGyB=XAy5Q7qk-d_*lHsH)oP^vuk*>E_C|k<+tC zEG zEst8&XQ?7rU1S)U*4pvRM*=z(7}G1#S(ZV|NP&SMNp0C;pmBh!6R8X+bk_mRb#wZo z&+JT302rfkb#Vkqw=h0|3XnaMeG1glo+Qfov`rh)%S`g=gIfzzNE#Oxu4gzE$=MaJ|Dn2J zT&Ap3MZEZwd})+sQQtkaB-WR7WXTqh1XOHo=2f_=tT)sAuC}vMa~^-#*_4wNF05~O z$9Y{3SKf`C=!abcbfiiwO&byvziuF*6Pa9J{s-F zD-xZpku=(Qe1pwWW3RmI+x^>j06ngo<^qwZfTd@W5qUuS2gs3VotKbD3(JrpgV$M` z`^Efza>oAV&sOHDM*;sK*Nw~}6WhCdW7AZ>q}kq^jZW6(lU~;4=IKdee#5uthR26o zmiL|#j(-3*{B3_ECK*%YX!Zhs9-xHuvOT?gTiL7PMb*GeCDm+CHNTI1mC~MevI?jQ zttY8!d#bvIpQK-~S~o|tH{fZ_T%}g1WLXP)!6w>#t3{+*VoMZ-zm=)4mH;!4VP9c= zt0ld$+%oXU51z?DZd=qXGLS{Ywub+P3})f3Fbix-CW?O^%FQ4vvXYZ2B~$u-qHdEZ zeVr*ym*|rnEow(~D*dtzEwb2?4J8};exg1m8(NeJW1%96%;#3z&?1XGnNTvJ?27ZsV?Qq zY(j7mSNea=6N!dm6FRj3Y9ap4pak}Fm6yyXYB>a`*V z6wb4q3`wzi9oj)bBHE*PRwG@J=Cd&gHkQBeNPf+;zS(BX3!C#{Pa{XJs&a-N$@|J% zZc2iUa)HhtZ0J7;C7KoVaVx%J!lM!EZ+Da$s)v8@py3hR@sJ;iHvYchh*$DZtQ+3X z?>@ZYp=cEx;Z}1fGbt{6xhO81MG7PDMyj0~Xbq`?a}`xAQ(dlmv}}S~v}a+skTj`m zMTAdfG9V!hTQy{HOn<JvhLGpI;Uuf^jYZov-y8+J)V9*q{F6@lx%lYAW*V9;elNDjbT7? zY~_cf%TRQg$Pb|uJyjf2P#RLqXg6~i?M$Y>UJjXwqVxL_M;D~x~5 zccdzvKcx}>9RO@gL(buz%40%(Ly-%VV};6_o;aQdB{o9-{zF)i9~3XthJ{2vrHjll z-Pm$Wq_2Lj&#hCwb`d{Le|u+R?-AQAD7gny_ciyo4Cog8TCJS8 zon3Nx&E=^)>K-=F(5fzs(RAuumYW$bl^w}9b~>8}2%ctH zu#|zHvLdg@z&EaDfp9cJf^@Sz-N3_V)knUhn3k^7CB-cie*Xt|1z9;B}yXvk1;naep zfHdt~iVGZAF;#br_+5YLR%A8Z?-Dri!pM=|t^3`_eATN%|1Sw$cY+JqL3wObqP^9I zy;956yvmRx?(q=$zUxrs2kr3yi?=N)XULONDj47 z1`U=eV31pJ=3+Glcn{Ls1)n+m92cMH zap29L@#;h!cEnlMsPj<13zPgjAWq&car^q-8k=Z$kd%MZMjBSxUFEUNx3ICfJRfZ8 z0F*0V>-=w57n_aIFH?W*&C}6F$ehmR_5jQ;6oJU?u*%2_*eq}&9j8gz4RAdgH8%C! zh5C}%K}mfxw^cilIve0&2Hr?olQ2?QkK48i>$}BlVim5F)^ane03$CCp02K;^0@zX zlOd`@sI-4tabX&&MgU`B+ARTM)XIjRshOj1EH^X`wmuez5DzBIsFh73jeChc6Y8c; z2#NZtql2_e{1Iy~|)r<(ZvvatRfDjN+O6)2jvDwZZ@V?o>pGH z<*tS$OfhYpsht-hYgrBp5X#g&Wt;x=seRheg^z#fw9-zen{m0_>TEUJ6rW~Uu#`BL zGU~5LoIB676R9JE)u%0h2~y7Xl!Nw66we37*+~l8Cb7Lxog%s1W%FQ7qRr9l4bnt1 zt>FsQyx;NW2fL*gmv@6-L<+W7f)Ee|c6Jrqt&$XDb26ic{x+#-g`B=Mh-0B_oI;X838JA+h zTH>fxHy4Kj4wPGM&Uw-^6Cz&Kayr1SYR zyZVI;N73dvmOu3(jA1P#3na{lP;7s$`(r=eRm$^YHs(6vIw_Lrct_LGIVQ{THkRX5 z8xFD;Z$na*NGn2%K?dUkDr3&?26ycC?&jk1S7S7q%4m?qC~7fupgED{V|5GXU%G-=sb|<3EL~w*`=TC6e(Z-0Zk1R!)i+`GtQN=A}^16jy*U!#{ue zggrF%GvUvg0HEK2|J0Yb;yx<=>7gG*(i9Z_eBeBc3g7-{E`r=_2fwu7KMmwX!=8`4 zz`@B3a+COPVp9SfhWdYPd2L()jpGJu^;o0JnqAguvQC#B%eieatIaGMHXFxX)@ZOs z=S6g6jAPaER6@$1wwyXb>NFH{Zf%&w@~>q=Ni$d?`H_>R)y`}w*^ax`ZiUSUZ@Vl( z;v-2drmx^5`FV*^H{TWk6;F!Uo?^ZU?ycqUd6N)E=KQMqPVaxX@m3S!!zKByQHnIX zXQP(zs1-qQ#G17fp+k8#9gW5Y&-%U@&JLonKwv-CTW*d#Bder->gr zP!Im87AGJW>RhZEU$NuFd<1ovnkM0e=#}|`=0<&6E`5Jy@o6ZaJ0B$LaK|-l8@_5N zNXNExY@2axdxFQd@E)*g`1kz(7a_CXXSv66((Bo1vXp4}WR-TGmIgVzaedbwNG zN^6&<+dOR-W}H4 z=TUQISfUXJcjr;@IRf3N7tTaL+0}YsiN}_mUEMows|QS9?sU8Juh>F%HQaVtf|SIY zTCP4IyBZZwiYa||b=Lf<`i^1?+0~iT<->pFT%GJ{g88yS+Q)kXWXi7I-|Xr}v$?~W zG98s%ofRB;DZjcrD%j!d_EbZ3h^FCQ&j}4V0Ak`h2o;3kAHS`{nJfLOB#OK-I!mw!cX($I9fSq_C<(Ywt4u-glC5gzrRGT#HOj` z81W3D{dcG`^~*EVcBWUq;>~9a_ARP!HMCNX;x5*5*TZ8JD1%WVA*?q6s}nw)Fk0J0 zJ~`xJo(C~iAjfCy_WRGb=QkJEnBj0U`0e}P_U=uJJ|f|Yv511_L=8l!qxOF-ERryK zOuVkTF*4T~O9RwtFx|);l3e7}ma=BeMni6Ltsw4kzFI6)R#Dxuie`JMtfG}Fu84nb zTkv-$D|fC+^%Wgy3Z}!*3Y-bfU~Q|YnnHUu@z}DfDLS1juKMF%XS-bWaNA`GQqq2E z#r}wD3Q#;Lru5YmS@Wyv8`*ztR8wS4mk*b7b*d={<{9TD?Y#jqRa5M7HAQ2Wt0`Kl znj$O6u+Mb$JKR2?T!*b;VoQ4G>@H_<$(ZXWB5-`$v7G3cT#H>OPAIc?@v+Tc3BM&O zWgz^fPDv6trY+tF;kZROE*h?wpYXX(7_kwD@<%ZNIMo;W>|8-XVX)l&w~;P@*9j zhPB83WE6J6C=ix6SmaESO1#81Xo>q+G`B56a}m)|SFP=J)l%u0I$2iN36)32rHQh3 zwoYwNHaFAeW~RrH%_W=r-WH(Lc6QaV%PtAHxo)$dZ7lrr$~uqC+-%5vl;5zB2R!us z$1x8cZ=vu(zSV)m%tn8&2O{wJSU�?tH64nUnhY-X7kg zKce0W-_bP>0^%|cUuXUJ^Dr>T@J)`*30cp4E*$xR#8C1hiD3D3!XlrwPC91b>SHzi zLg(b9ZC+}bi_OmI>1nHJuD5IWpq#lpcz|+x>B^~Bj^U@o!sayNBgU47-?c-S&0FEQ_FwCW)*&rbf^rMAK0+yU6jeG zeFHU@XRx_jaqb7AU)Zp$&M|ade+E{2T&AVBSb44!CYC2qukX3fS*}UhmZ>lO$waqY z>LcR!7w2-c|9Hh{VaFghP~f7@0tf2Ur3Z>Im_dL-{vtu5$Q5cV^jFohFS8ohKK^Kp zxc4CBU9NuzH+Z8E_bVwPD%hb3ZjDwT4{rGo@Uis8<43^N2C3CyYq7AiJ(CY+{BC?O zIQxP(%>6D0T5kFBTA;rwkUJRlXwodYHdL|bH-LoHDB>Z9QNSG!MT5i5bmxVmK7CYmBu>exNGL;xWP<E-X-ovu;L z(d-Q}6f!5*3Vl>2Lt)7odc}nJa9O@Tqck`T3vPwn+gb9mEB_9~S&1C1PzBd9lC!~6 zQ!4y=d9DuXl0b_m+YwH;BGxa$6%m2;&B1?4M&PZU)$we{!jPOkF(jeBO>Ov#A$$_* z=0d#@6F;C;>J6Uti6FX-b6Z!)T?$#!H+$Cc+BBA3kS05@h5Svfp~C)LCeiA{-b`nz zD$$~MMaqAP=OPTQ6dS_WQ2+L_-qgqW5GM+2D$_tSQrsv*D}ep1XXQi5@!;k7nzwbB+){7y(G-HIS-P;^ejQre8b%! z`A?kpT{ifo+wHsxd#-gk#l03-V}@k1waCzJS@r``6w}#VWsT@fy>L2#gGpiU zzW)RTPzY(Pr2Zq4t(HIJeSBmWzEbT(m=V4{c8S}U65r>6RAri?=kS-RZrKN zUJuhuC|ReF40)2{09gxzfIokiihgIFPx6IFd{-C6GM$8QlIEM9Y(TV$r1|El@VhM1 zlJSe^1Ldn}Wcj|J);W(u2Ng#&Rv4Hr@#$R2D#=)VS1ViLm8j%9y>!yoPWl&i+9 zWX{Y|TK`VV78$uS)1H5@`HG?$kCX=ruKu9S`VVkL1w=zl^xuyiP;8uJa*%!VK4E`K z1a{Kz@sOR|)9;BrTi7SZxhKq|g_bE_BA#d3itRKIN;%0^36|F4EkNX(p5b&-`x+NL z?q>$A?&kY;#j+6jtbxTIs!4;gmL?_dBGDj1IiN|_&KL0y|Ga;)apSs?4Rj89)`Ife z_rdL5yg4~gZ7qcLC>=Zgz#R{or?83RdGf*!VMTuMABiRvF*~W^K#yfG=8=5j&u1AU zU)Kua`m563pC0;AQc}7eRk+7U*AE@SjHiB69MPIy>0X0^}yK)@;Mua2V>h<8Nmlw5G1M!iA8_zK#7JG$dFL!GmZt z2nZDf!vDUg^^E5$5?aUfN;sdphWzH?^nTtOT3C-591m1~@%WwwQmzsy^kbPT^eOyJ4LTKB5~?CtH@9a{-HcZ0!FKaQXq=uD9CbxdmEZ>hY{` zwQd02`f-2eaM$;EHkw=bXl9@1cmTP;0oAOS_ugiq#KV|_4hQZ2HTE}#!}WQ$SluNY zUo}@%*ZMLg?he}5_H+ZtG2#6Xu0PSGzvUUIRExien6(21MyFL%csGZ*u`X51<8b+S zusmZv|J&8Y`CyYlNBkj2$NMt%bnY(nbdFax>g#{}QeS8JM?GNb^67D-H~)$)^mT^Y zE=!R52dAvU`}cK5y^~%_-q$&6a;?zEo+d_JLT=|<;lEmSX!mSJczeV7y=p468=XX{+n%hmq0kZeKXnqQ0g8FWLmn7rC~W5ef2 zZme{!$iK3ryhC6Bd$>qjzcWIwbt}_hmrjdN-qDDiFVVtY-7?qNg>5uuNSS+Y#wsU_k1%kay(slG?Hny#i8{O%S zS;bo2k=ghwFnD^?u?RSLREo1G-X(;|_IC%3&$v>~o7%C1=4VK6QacLFC(^W)iT8iR z{gmckMH6G|+m3>*?`T3SPKfd>qPyMpZqcxmEJ13wn+nEc%;gbjyo1*bQ+K-$PYpVl z;1RDBhH_Bsi8_pH0s^jTh?c6-YtLv&BfV8}B0S@1Fv)?^C zk^LYN9_PW6!%`$>gxzh+eSmaECNh6U*e&cM>d+!OHlAOmrFZv3xG}wyvJo#@1^DNc zv|6vPX2BY!rD~S862-bPck3rv>Ug1=<(GhuVRX=24GK!+%vn5m%&csQi>nrgUZ&Lb z-WfahJx?H&Kvt1Jsn+nz_opN{pCAzifr?7 z%_9YEaZM85gWP_}3(~pTa_oN~)4{0Lz8BJ>zx~GiA97E6aee+)>wmR`*sfuze@p~m zjksL0-2%eQT%$i+m>(S7a3U%bw&7iM4^~#MSTEA^Z|1^POOBA#M!wa1`c`-6*W`a&k#F_E!h{eX z>-BmKc7Q(A=NdDYdhEdQFuHEKq?ndei8nRbfj^DZyD&XA^*&nDXicLvZNH^_eRX^B z4+Ds_8)N6Qo~Q?yK#M0~Jqf;n5+z}~HWblJ2vxzStazM9yFe>HTU#suyAvn~OgS{z&_jRq6w&!=(&cd?`6>@U zRZyei$RE_B)gc;(P7>k~877xg=J7i{xxj<8;D)v>?4@odP^2MWf!5(_yz<8I?S241 z!T0N3eFan;OA{`O1ozrzWIGbe+MG@ACdadsTWdu4iF@j`jU3qfR z)Zub>=8S%Z|Ih?p0ZeW_d@VfO6DD5}juza<+Zq~xKKMuk?pP<&Mz}hVZGp=?Zx0xc zASN*gf>_(S5cRZRyB#uInHNjXyxuUa{cYh_81uV*5cN!(*%#PiX0%G3df5 zA=1;h?`I4GeCcKGzapYh+;8h<_*Gw});QDfs|l%FCxivP zCR8?6ULYD`J)g&nM*W@EE+EU{Da15+5%{d*Gp-AVKkGF8{ zU2_4Up;i5lyxSTprLV?>xs+C%C8FXBqJQ>e1{9@-=3T7!sX;vKMG|H{%8M!a!AD`f z$c<9pTqGiPGgwQ+UusrU{m`6Smdt-BT0`ES(gotY6 zd5rPHp$~#T*DyAViupx9OftFt{B8aneH1@w&LqDBwm5dC%A$(4mv+9Rc}(*=@zSh9 zZ~!#IzmGrcriWpAXU&qw(C2hBeq25tblfD1CVoMm7%)!K8(6UwS4Z$s;pF?R{5tHS z0t%Q110N&-`esQ_{D(MzTQ1I-W~ZQi>Sc0R^;|XQ)LWYU*=9FOpR4V#^{X_oWQ;P8 zFW{PAK_3wb`U(+K@A3s(Ut)L>G-;YAz!6fs;#%WD80Rb!b zO>+H<&Gq&^*Vul;|fNY{&HQu<5oi#?4sPY>F!J-hJ6HLR$B0ugTf? z7i{Vm>HQj9UTX4h#XPh&V2|cP?L#3qXsLWF3liVLcWOkf32f)jB|jNN^ZuG7R;SBu z&Wn!j2(+&a)Fx{O&Iuh4VPCF*X}UTc=o)qoyL_z#i!X*jr=rzA(Xg z#TboSp1hDcpFN%0<s$NX)#sedR+UZh_je~;g<5e6Yl(vrS@k8gA&&#$tK273c{6fvYW-JF*YcEf~ zWPUY->3#ro89n*)tQ3oa_>eOxgmH%6C3M<|YM`p^j8TU0yhEqC2^1F=hv&hvt^?J= zrY|ke9aM^n88J^RWrRO=e3n5n+-Em7@N4EjWyr{ybpd!@h0aVaM}DS+zhLWbqh!OE zc&Ju*xz~3(d1R;Is3VNC>#PjF&5&xDDEJ(r9y0{Q#TU$QRa-2~ZdUohn+#4gl2*k8 z;B0{G-+jPIgXumnD+iV5x4_xF7iU(q%bjJyt8Yh3Z05aLag$9&gS(4VD<;okhzWs3 z({xaK7yk+~wMB+hJ#yM`Cdhl5i}bK0pS&glbzbF#$)zXkkGl5@EEMt996U#}!??c> zULtz}0sx9@Vf^9Q!cqSa*@0Ddt@c*2zbbX^^r0&^tl+U5&!JQK>d|PYC#oa42XO?$ zkwGTnhPc-v{dGmX2ExlA+=Q2PB7F~4Q@Fo$p7yQeXQj?Qa{95LyHuEP>sV&*>-#m@G=Hs*8R1Qs;crw}jYtIwL8QScFwB-g?~dj#{My zr%g$x@D5he@=q`;c!FpCos%mZDI9Rwj|!p4ObT)nM!3t`cZoQlc48Nv<$=jaO#C=c&Ukx{s=OOSpH(A%!k z_pY$XdrQmad6;s@Bw!P$_l?9xbXmXwaDPqcaa=6gfKMQy*sM$IW=^VPcvsW81Z7ryhDu?a%dfO}jX_DWpb z{PBxa>IN(wnu2*7@UhUW1!@!V=zn^1>Oyi9yDR2M-1SN2x7=Zls~ad8DV9`Gke^M#(VxrobOX`m?V-B`%Hwo=SddgY=ITx0h+*KIA?he3sEJ2=-QdB8}^< zvc6rtEovJa|B90&#>EkggI@&`{ddq?9##+Y%OVR{I_&}ulnfjx@Sf}Inc~IRpKiN6 z3Gq4}Ck^166YT)2Zoun~Llx-tX|BWgK(PJdr`aEM(C-m_V*`b;I>%lv;JNW2xsUY5Fj&uXDK}5UXW@p1d-wwje_1c%G>(=s5 zkFAl;Zu8GqDOp*My?*2mUXBWNJ{+|Y=_>>(RXID#Pef2FZb0q5EG|mdN(aKp@bCrb zo+5o6FBstg+m3y$vTVu4>E))(vjL5R79puP2i&__)?2wixGHS!vY4m^1KKzV^PsM7 zKcw;Ba4Anx=0R~l_kpWmL}}U6GlaJjlFp!V5=s9WnHG$@hKeAAgG69%H*cFv@H=q+ zTE&se$cJjadg)A^eru&s28aB0_#rq>kp}F2;cULdhr@mT;`8F{D^io+oj8hRQ8-}= z>_k}M2^9m^baz zz*G>evMa4?*Z-Uqnanc~ckl+jIZQi5j+!-IaP%ajgT zu#dq(`)CG8*XS{j;q_3XVTWi?p~{3b`R9%>GMa>OCWYs5ZXEv*xIhUYWWW(#9dj)B zmGF+UB9Jh5?dbXLle?J;o|KLPN@-a&aw#D$B^-vJ+r zD^dbmqJTSXRVrp`hQxEnAM?jzqDks2_DT(UUjY?+Qn_bp1MD~Ppn$DlPCt?bFINSBpdnAgG!QYF8dun;F}zTO)+Fdnnjlr7dXkC2fH*?} z82`5pvNt?EEUtI?hzKlhK=_+|ryY=k$G{7<__H|4;AQA`$>@T&GBgaPe(xjwKkNY7 zA@b&~LItqyfrG;cC$`Psv(z=rlD4ZIX}Yl9Ck3735#xMu{Ne<&l!>ua@k~k8OCFp$ z5{aCjmczoE49n58Ii-r=rOdWdRaQq zFrPdD#Z$R?Ly&`m99W|J&z@7;9hTg0DYvLaS0@_KUiMPj_V%Ckaw(nV5)y1|C(9MK zOzT;HZ1(!JXrJZQQr}R)XKrcp&hT-1&gmbOtL!f6_26$9kf~g<2?7)hwi9y+>k|4( zJ%fC)9RziskQup9sMCk0hlag+AF2F#BltpU zAu2S=8=PnTQQnK;S@+V~W@J<13~ihs+M}1>=uv4vk}~?7)zPGms*ybHcw89&4o0Zw zO@pJ$bZ1iOuZ2{(IRNPxD@WNtE}iJ37j-YPH|zy3vX9cjw78{zaO`>XwVya+8DB)^ zz{^4|#-^N){S`J}Ye@A!$aEY?Qzb&!}s_te=S+|?u{`Jsm{Bb1KKG8N`xVV-bmO>r1uil!@hLGHco~&Pc>coX{e3IJV zLw21{^<%B}e(#l+L%l8NCSA11Xp;i#|GW7ml&h2~1Uux|j3dly&2sRVLn73fZ8!{0 zL5rADdq8*M^6-W;=}B2#M{x-Fk->lr*`DXw2wUYV2BO{W-BV$)6KLBzGtNQFjugZV z-H-n3Xaz5N{oK)w^^~G+IA71*^VlE|I_#m=jjQxNo2`MS8MasliP7f?kK;nhv>&< zi#a3c(%Ds{is70V94T?dGSh@f?!(6uP@eOyY3^CgNaVTnpv{`;kijUqrmeeIunqGH z(X+1e%B25ksg`-N?zQrX>u*dv)D_5p0PvU#OX)vP)qL7Xblf-z$^pJi*MM`bQb{uy zU3Sn%drnNi@82iyd`G~?W1s`A>eGw8$JTBFH}^h2@|uk9#_~?gj+wMWG#L`rZo@V2 zI2M3Wn4}D>Wk!A7lh#H^m)bHwsYM{Gnu`6YsxC4t7~8>FfLZ(zz&iVtidZI_E~jfp zPa!S*S+Xw;H+uUOtcGg<_|>#O5@TiKlt)?Gl=m9$(ZRz#uC?D-nDsEcShb%iA z9Iw}QmdX;fO>-cH4o8lFH4f=S^so+zB(XCuiy&-v+n*rFbWy*I6pt5uRcX0|KI=WoOWn@-t{1I0t3x8=_^Us#?=wTb;azTDErI_PKp?!=`gET4VFynZi`DEp z#vxBx+OM`Zl!i^+|DkSSd;y}bk4)+;3xTeWa%*GKCf6pEx&{G+!^uaIWX~DKE`5l=5 zYDt$(5yP(mnz|eLQ}{{mzvlK9p}-;KQq}kUVlcQ9v!#VC^QvH_=k~!HLZmdU7)Hzm z!}Np-3=S4{7CJ#=yXwpMDa=5?30gh==c4!(NO@rO^AEd3McLTWMgzk6Gc>S+k)P$1 zn&rUp%pXmue9E3HN{VTJV!04Q8;)DlO+5!^++bomtRo2x%V3og+Pfm1QR*=^K&q}`Zfr+EkjU$k1Upve9 z+F>2a@w6D&#H5DtBAJ*gcs=Sl_2;yG=GC04E&l}YL-&xhqSXdCO!{_ z@Kl*7)dBLYGFYEkrE_s*s9hpR-u!xr%;6GJRa(~pxk*N6?*vFz-cLtx0WT*iW^D9q zsb5!ta}TI34@A74Mn0Y|mcSy}0H_mTIdTh^*%Hfu4;naA$< zTN2En%Pux5A$7yc6+}r=S$}x#v3GJIvazuN0+e zSSY^rVpF?krggj{fb#j(3nHZ8XQ@x&#UGJo->b-{B8_5!%TTGv4_I;JIU}IkR)kn1 zFjv`PxA7GJ0+&WV@S2pgETnmDNb9Pw~J8 zLq6yw_Eqvw<>!a_iy_wq#wRMQ%XRqtmcY8{mbl>I)eI|61W1}sonUXXQ<|sTO`5$# zulsS|N^j<>xaeymPPuUZ3X)%#CziUCzhBq=K>zD}G;{lOo%V4mbLu!(%A*WwgjW;& z`f;t)f4rMX+V$}#!9oIs%MQoymlz05^}-^z84pcj=&W2G#DlL?srPdiDPgpU#m^(< z{MH}Bz0XK~eF99VSl=s5N8_IOJhNbu_e$XEG%&6^u_c-RR6OYm{OmqaNLRUpgd#W1 zm=OD`FGqp5i&@rGWmX10i{_+cS0V8}f6&0O{?xsb)NS@Q1{g2Kn^v9mYth4!3_2$^cY zq;yTMo@QV4_VzZ)I$V}7jW`jc#bR`5*jD z{3Q3;6t~$QYdJSR=3|s{3d>(5Yl|FW)+#mUv)ygqa>3zm6NbYe(TPIi%#S2rKfE6? zl#&ga1RRE}D!2n_iBIV0x6Nm-dx?@(3E^eBZiyH$N1J+y+!zr{;OnBgY#j6B7KN#i zv<$RR(thny^kW?hSb z|9qKLsZl##l?#*U|GSWA ze)#1DbuYF#BwZz>KQ;Mod}{dQU(>u7LvnEU9)VzVu2|P@!3CCevuwd<7ctgJpEnvw zbT&A)o;*c;c1M^{(8&`e+l0eTU;yhg0Qn5awD@_@-q6<4NvhsKJ*x&0tC+!*Qlmtu z${g}j*tnFxaj^dB@_lAg5mG%^?*6of2-$;UUaw*$u2G%h6Qg_g^Rq+cP-@ubtk@2U zf#I5=fgyAd%wL8U<9L$`2tucS5h^M0LgDaRh$Ap_b)udhCT~hkKD`2x0cAuJgvnfm zk`EQUn!0bQadeq6Q}#LQVJ-3*38Q>(3yQkGJTOayQ1T%55rAeiERC0jUhCTv0f)fbVt^ zn$rBAv?2sB%BMLk zDjMh7l1j6a`KaIT=RR6fS@S)m@pC0r&!sBnkr}L^jl~l^oJ2edzkeO{=`BFvLq)1h zDwm=M#+U2_cr(HZE-uSW-O4Zs@)axjUOYjR#gk{%raN2i7>a@Ie@F4nei)eSR*i3E z+NFMN9h+VJJ~_CPd9)DTmrdFH6u5kTF?uZYl%l`2K`>+Ie0FuBnh6B(;{q}^*X3P9$Ob!W1loadX{?*kCITbuQo+V+=lR-N_^nN-W6;x+vbFIy-^U%stQ+x*~lPANYw+oZcng1MA`G(i<8~ zbJD`P9gq7%tQ=>K$ftZAzs7G-(y@0`R?=}n@P}!{F_VRprBhr$8?Hi3eXa0>9b;mG^v>51(bwrjt_x_(dgi%mRt zk=5-gl5{N*3B6`|c^2oudMD*Luez?6x8!hG3VJ=lKR>bM8Ds9h2^%c(h`>i%S6{gg zXtSvcHdoUpdR&gg|Fx!L>l4f%mSzGE$M)tb;>iv#;a1(c*nL6xqfNj4cLH?7$JnQ( zybKM1-EXZr;A447GRHH!ar)!s=XtAw+Q@8%bI=h3dFKWT5Mq?QOSi6_4&0R|HQkwb`?;>kB`B5*s%8-uY`VP{%2vGx;`!Du5q3!d+Qd@5-VolY6fmw)%SV80vKMZRkM1aqf~bCJ~;;&T>>$&%hCVJQGLslbM|o;4J7 z5YPjSk{84AY0T@$Y4`QSdaOWp=)MhyYN>J84-4h2f@Vc-yr7>{eYR>`?|(%6L;#98 z_RYM#ngjZZ!#+Ugdn=iDGhBwRQgYu6H=&tt5~~Prl&HG!P`mDNlxY%{1!WiWyXSf*$!a zw0tdpSbJ8boDqw|%^#D#Ca#%BJOS9>n|PTY&uvVEr#_T1$%39Pejng8x-ay3@sDm6 zu+{(6QCi6!H?S}mfsE+kD0tQDw2$k(!VcJ4?i%^6JgkKciXme;v#Rtu)|-XenCcDM zgL-|{Xax#)24<2MyzA0}(zN$$To1PD^j%$!W7v%#?0EdzQ>AB-sV z7EnCRI4)s}`l?c^NowIyUsoCukqrG0!YCC2ITGd(oLc02ZkS@ zyxDnaEh{p8oP8o$rLzb&Rx7(L_U)$;Z^x`4KuhCzi5YBe<4V@Eq}@d^jQ3vHu}>Ek zHmED_ASv7HOIFNDrC^hun3MS3f}^3F+}DGOaY@^&@+Cr<72jTb>kVLrxLaCLNr8Z9 zAod9>fLXVzF1)h1^<(g`MvJ$;g8k}y?<<{R%ej))oR7(UJ%~JsD_b8j4040G!RK8p zLOAEJO)huc(`IVp@GPlq=3_CA9Y zzowJI*E*iwGD@-Lq9nH_^)7F(SaVZ)*+50-aJ5+VW$LhRfCrVb7a2Ui8iy+_4U+R+(c*E(*RvTSb``bRsMxfP0q^S_kt3rpMgmds zY!(li#_FH?Ju84# z|Etr2*a4VvU^?!R307=e(V?v#C#h!Tv+MVnpEmsVna3Z52bH7J-b^O8jsJ4wDgU`& zID0hGxp{n;eD-x1r;znLN&hT@_1u{pAK5rz3qJwj4%aDsl(V=GsaHMw>!=nMLKVWm1}Cl z^UX5Q9$=F!(InwGGR(hMl*P#YF+HF)_?Lbi`eU{=Ed&>4Z$K9IwJzlEjrc2OwrzGL?AekD|V&1IOc-EL! zWhRwlp5LQr#M(LS1EY)?3O`DGy}ZiEIkQG*L>8Ivxqtj|hIy69omM@R-px8nJw{fV z4QSQL%=UgqZV|Bcv`G<|EZZq7@p9~(g|2DFKB-swifTQ-e=HR*m=Ul#k$&Nv?)-^w zFQp}YdZys>ygC9SJAm(I5N5-yN+}_Q$hs}pFZ@OJ+%TT`^(Ou$lT+{`rORx7n}@UG z*h3ooSm3#F`r;{P9Rab#bekW^;0w;`Z-AwHiECQEDb{uKK=D0gIDQbrwO^W7Q$phV zHH{iCq7~n~6jp27z}3Bu_SR&U$ZVV&xgG4QDbxqs2WM2;>Z>Gom4 zc~g3@>{-^UM;54Hp#@QuEA}ob%Vm%E$_?Hbvo122>-}RJNY2*yA09nYBUws zSSmkEPB`$;*36V&Dsw<2`zm#k~vn{Yh?2uVy{QB=T=j{vGg&buv&k zKrWEcbgnPg5qTLe-^tuw^%>D3>&LJv0o1R<=X{>w>j$PTjSn_*gRWUH-dDWXUp=qy zdgnUf;CPJ{8Fk|)t@#%fvnIWKzH??l#+ykh<8^5(UKJG9>sTGAwFpXZxB|`7{0csO z_)~ftf1c`DQ-mL+P)Y%XXZ6uv0!&??^7#zR*(gdQ9Eo zA1$;+b(ZfDVT{mg zpeaXkpQ82g>EifywC`#a=$oZ$C3^Vn>)=hGhCYfMs~F~jo|Yfr;ULL`V`3xd?h$nN zVXr%gY&=tv6q7pOtBhdTZy{5&sZ0ggn8gCd9#QE;S$3>2>w)LEE{glSAq7?Jf-j= zIwz5)NoUN!W#*Y{Zh1=H8y?V?me5Uil1mrHU$LCYSBdjACJ&viCPnSnWcO zZ09`H0O!0;g`kZCKJz=6N_qgM9z0Qjn)WV8T=4Qq>k8-}a`tm+Ie1{yh`=S_6rl$ejc7ebm{JqIswm1hCw^S9H62YYr|X@Y?)srttxc9esr22rHsbLlox#B>$a|zfX;yHO?mXp=+J5g)`(kY&Utre zmfm0lohUA}I&@bPRskcBu6SKG+v9R420M)6R$bxOa(@Yu-~ij$1MexXY~|24_CJs~|!(Rx=? z1KV1tpwQC*LGS18q#BRDSZnhV=2GZujv3?%y_>70*P+<*ux@j{Tl^aj$*JeMa6$tg zww67k2~$~DMuL*X2?7m0kTI=qZ2+8{EWp@6R2^}VQDo4}V4*2}T=|L0|K(RnX40?S z;&N|zk;h|V$P0rw$>!JqAKt|Kj)`O#r$3?bWCF0{KAw2Uef<^P-D@vG!?;eRko<@uqNiP+Ki-+?P(|QnhyVJAiuq<5CX3 z@FeHvIF*;l^>kdTK7zIouGXIr4ZPVN zFA75oG4r>wjLwN{sw0o)-$xN4BA5Mi6EH0pn<&X%g8x<}J0yE*qtjw=h2QhGM`My8rma0BoUjqTyHo^6%7Mg1Pap`pT(TQ{!(v{$OOj}-4z8mJ5l zT}{Jko;H3?Me*egxmweFJ&&euZ_shFuaQP;$+`F(L{fnNV8we`^1}zNxDoHdz4SL# zDiY>+iE%(ytRN!XYs!Xzd63^qj0fRw$U(URiCs8yp?Hux>bA-H@g49(az=b?v+{sH zv9~!nvbS0D6o2B^TX{(bvQsVy&^oe~`OS^m&wwJvc0KHcy?K2xM3h`n0!T=inqGc; zd-79r<;9B;;%t)-{7~1PKr#8wNc5Zyn_zEap1S2}PT1LMq;cBuP>@h$hQU>nIm;nK zn2>BB;@5qmBDXfD`9grT0ridadT90FYJ!-n9WF&Pkq_BrV#r%!GFSGovUCTLjqmTO;<5weK-(ANrl%8??)0xeOW_(wecY z9fQwJ9(PD`HEUIR&UMTU~2J747>KRAsHb%ZXeX;qL^aoBWlhez>*#T>N8U;3N8Y_6hU&_q(Q z{u)aX7IS;BFU1&O+je@*zS_ND8v5SwCG4(JudrqC7g$XZlEEwA4LgnOCutA;SMF*3 z$(08E$ujuAwR+TyG#h=0n{}*MB){z}zM*C7S^W|Z@o^FqH9&G@O ztir&#FXF(D3aN_V=J@Iao={L?04&Jz`f|H5Lodc-9=X2i!K)3|b)%|b;r7}`*sB$1 z!>CZ1{^Z<;G{=PF=O;~&q_1HFE!IjABQ?{4_m@L1s~jr* zIBwVXi&DyfI1zc3moM+8+9K>;liJ*y3BK_gh1ZPJPuIfXj>&)zWHEf_PycOoePu1} zkeIrsDI#oVTisV9WKH9Nwjj+0c|du@az}pT=Wx~}u8;%evT0Oa)VKx09QXp8o?AXg z%;HW)c-EHvr?Al;@pwr(CNOZahBTz@EI`k)O=+at4Do)O#n~vL7#L*;)pL#Y zy(WEW4%8P@>*JD6m!4Yr@k35ZQ* z0zHWo8t#|HTO`;Am8=mcJyeb7;g^ZUYoeY_kD3|x=G6fvijs4Kz2zl4E-L4pyiw=$F{H!gMu!N4pb!oZM1QTITDxvJwAc z>+~Qg^1l+j4CoE=U&%`qqyYa{`mO45(n(3rc^r-wgvZrvic12XRsU!Tx_$LACYw;}0tDgJ|l5nEz(s zuMZ*t89*}iK~#U!U4POfkQIFp<=^x(Gz~jH0wFa3{jC8+!~hBeOhEMh1HAtLC)1GB z|A6K{K>0jm?ms}>5DHK&Ck`x=LA(q>Xn!;Q3ob4v`mg*yVE7+q{0gMl5JdBrHS!0r z*h<9MCc;!#0>Qq7`@d}gf^7snr?;I5+981`7=h3z{?z*HZHa*d9}J8gH3$a#zbZq& z{XwD-7b6fQ)gKZiCm5j#3erGZ67~O(sz^W(9a0d@-*Wy{W9M57DX3;)xS?F!|KS#s zLg0--RDW}nS@&06gr3!gvIzf&l}ZMJh?9eeAjZc3*4*A0TJ92ZNTl(Myd CJOxJp diff --git a/docs/e3ap/integrator-guide.md b/docs/e3ap/integrator-guide.md index c6acbe86..c9c78269 100644 --- a/docs/e3ap/integrator-guide.md +++ b/docs/e3ap/integrator-guide.md @@ -32,7 +32,7 @@ the endpoints. | `ran_identifier` | empty | an empty value is sent as `unknown` (`e3_interface.cpp:153`) | | `dapp_name`, `dapp_version`, `vendor` | `libe3-dapp`, `0.0.0`, empty | empty vendor is sent as `unknown` | | `e3ap_version` | `1.0.0` | sent in every SETUP REQUEST and SETUP RESPONSE. `init()` returns `INVALID_PARAM` if it is not 1 to 32 octets (`e3_interface.cpp:243-254`) | -| `link_layer` | ZMQ | POSIX if the build has no ZeroMQ | +| `link_layer` | ZMQ | POSIX if the build has no ZeroMQ. URING is the POSIX link driven through io_uring: the same endpoint, sockets and bytes, so every POSIX row of this table holds for it, and a URING peer works with a POSIX one. Linux with liburing only; `init()` fails where the build or the kernel lacks it ([docs/connectors/uring.md](../connectors/uring.md)) | | `transport_layer` | IPC | SCTP only with `LIBE3_ENABLE_SCTP` | | `setup_port`, `subscriber_port`, `publisher_port` | 9990, 9999, 9991 | The POSIX link uses `setup_port` alone. ZeroMQ uses all three: `subscriber_*` is the channel into the E3 Agent, `publisher_*` the channel out of it | | `setup_endpoint`, `subscriber_endpoint`, `publisher_endpoint` | `ipc:///tmp/dapps/setup`, `dapp_socket`, `e3_socket` | The POSIX link uses `setup_endpoint` alone, and its IPC path is `/tmp/dapps/e3` while the field holds the default string (ZeroMQ shares the string, so it is mapped and not changed). For TCP and SCTP a `tcp://host:port` or `sctp://host:port` endpoint (a host in brackets for IPv6) names the host and the port; any other value means every address to listen on, `127.0.0.1` to connect to, and `setup_port` | @@ -62,7 +62,7 @@ in the connectors. | ZeroMQ heartbeat 1000 ms, timeout 3000 ms, on the dApp SUB socket only | `zmq_connector.cpp:30-31`, `460-466` | | IPC directory `/tmp/dapps` is created with mode 0777 | `zmq_connector.cpp:102-127` | | TCP and SCTP: the E3 Agent listens on all addresses, IPv4 and IPv6, backlog 64, unless the endpoint names a host. The dApp connects to `127.0.0.1` unless the endpoint names a host | `PosixE3Connector::listen_on`, `connect_to` | -| SCTP is a one-to-one stream socket with two streams requested, `SCTP_NODELAY`, buffers and delayed SACK set before the association exists, payload protocol identifier `0x00E3A001` and a stream chosen from the kind of PDU | `PosixE3Connector::tune_socket`, `write_pdu`, `read_one` | +| SCTP is a one-to-one stream socket with two streams requested, `SCTP_NODELAY`, buffers and delayed SACK set before the association exists, a retransmission timeout of 120 ms at first and 20 to 250 ms after, payload protocol identifier `0x00E3A001` and a stream chosen from the kind of PDU. Linux's own smallest timeout, 1 s, let a burst of large PDUs stall a send past the 500 ms send timeout, and the E3 Agent then dropped the dApp (7 to 8 times in 100 runs of 2000 PDUs on either socket link; never with these values) | `tune_socket` in `socket_link.cpp`; `write_pdu`, `read_one`; `UringE3Connector::prepare_send`, `consume_into` | | The dApp retries its connection 10 times at 200 ms intervals, and opens a new connection for each setup attempt | `PosixE3Connector::setup_initial_connection_client`, `send_setup_request_client` | | Lowest free dApp identifier and subscription identifier | `src/core/subscription_manager.cpp:58-82`, `211` | @@ -103,13 +103,13 @@ in the connectors. | Claim | Code pointer | |---|---| -| POSIX over TCP: 4 octets of length in network order, then the E3-PDU, in both directions. A receiver refuses 0 and more than 65536 | `write_pdu`, `pop_stream_frame` in `posix_connector.cpp` | -| POSIX over IPC (a sequenced-packet socket) and over SCTP: one E3-PDU is one message, with no prefix. A message longer than 65536 octets is an error on IPC and on SCTP | `write_pdu`, `read_one` | -| The first E3-PDU of a POSIX connection goes to the setup path, every later one to `receive` | `PosixE3Connector::pump` | -| Per-peer send timeout 500 ms | `PosixE3Connector::tune_socket` | -| A POSIX E3 Agent removes a dApp whose connection closed, with or without a message after its setup request. ZeroMQ cannot tell its peers apart, so there the idle time removes the dApp | `PosixE3Connector::drop_peer`, `take_closed_peers`; `E3Interface::remove_dapps_of_closed_peers` | +| POSIX over TCP: 4 octets of length in network order, then the E3-PDU, in both directions. A receiver refuses 0 and more than 65536 | `parse_stream_frame`, `write_stream_prefix` in `socket_link.cpp`, shared by both socket links; `write_pdu`, `pop_stream_frame` in `posix_connector.cpp`; `consume_into`, `prepare_send` in `uring_connector.cpp` | +| POSIX over IPC (a sequenced-packet socket) and over SCTP: one E3-PDU is one message, with no prefix. A message longer than 65536 octets is an error on IPC and on SCTP | `write_pdu`, `read_one`; `consume_into`; `set_sctp_send_info`, `sctp_ppid_acceptable` in `socket_link.cpp` | +| The first E3-PDU of a POSIX connection goes to the setup path, every later one to `receive` | `PosixE3Connector::pump`, `UringE3Connector::deliver` | +| Per-peer send timeout 500 ms | `tune_socket` in `socket_link.cpp`; `UringE3Connector::transmit` cancels a write still waiting then | +| A POSIX E3 Agent removes a dApp whose connection closed, with or without a message after its setup request. ZeroMQ cannot tell its peers apart, so there the idle time removes the dApp | `PosixE3Connector::drop_peer`, `UringE3Connector::drop`, `take_closed_peers`; `E3Interface::remove_dapps_of_closed_peers` | | A POSIX E3 Agent binds a dApp identifier to the connection of the setup request it answered, and drops a later message that names it on another connection | `E3Interface::bind_dapp_to_connection`, `admit_dapp_message`, `last_setup_peer_token` | -| A POSIX E3 Agent sends each PDU on the connection of the dApp it names, and a PDU for a dApp it cannot tie to a connection to every connection | `E3Interface::outbound_loop_ran` (`target_dapp_of`), `PosixE3Connector::send_to` | +| A POSIX E3 Agent sends each PDU on the connection of the dApp it names, and a PDU for a dApp it cannot tie to a connection to every connection | `E3Interface::outbound_loop_ran` (`target_dapp_of`), `PosixE3Connector::send_to`, `UringE3Connector::send_to` | | A dApp that lost the E3 Agent registers again every `reconnect_interval_ms` (default 1000, 0 is off), with a new identifier and no subscriptions. `release()` and `stop()` end it | `e3_interface.cpp:1489-1538` | | ASN.1 encoder buffer of 65536 octets | `src/encoder/asn1_encoder.hpp:39` | | An empty RAN function list is omitted | `src/encoder/asn1_encoder.cpp:316` | diff --git a/docs/interop.md b/docs/interop.md index 07dd1841..6749bbd1 100644 --- a/docs/interop.md +++ b/docs/interop.md @@ -45,7 +45,8 @@ No peer is run in CI. What CI does check is messages the peers wrote, and the by | Peer | Link and transport | Encoding | libe3 role against it | Status | |---|---|---|---|---| | libe3 | ZMQ or POSIX, over IPC or TCP | ASN.1 | either | CI (`E2E dApp Integration`, `E2E Topologies`) | -| libe3 | POSIX over SCTP | ASN.1 | either | `test_role_pair_sctp` and the SCTP cases of the connector tests, on a host with the sctp kernel module | +| libe3 | POSIX over SCTP | ASN.1 | either | `test_role_pair_sctp` and the SCTP cases of the connector tests, on a host with the sctp kernel module; CI (`E2E dApp Integration`, POSIX and io_uring over SCTP) | +| libe3 | io_uring (the POSIX wire), over IPC, TCP or SCTP, against itself and against POSIX in both directions | ASN.1 | either | the POSIX connector and pair tests run again with `--link uring` and mixed (`*_uring`, `*_uring_ran_posix_dapp`, `*_posix_ran_uring_dapp` in CTest), `test_uring_connector`; CI (`E2E dApp Integration`, uring cells) | | libe3 | ZMQ or POSIX, over IPC or TCP | JSON, Protobuf | either | CI unit and equivalence tests; the live example runs are manual | | Aerial | ZMQ | JSON | dApp | CI for the messages: all 17 of Aerial's published examples decode. Source for the rest. Aerial's own dApps can use libe3 as a backend (`e3_backends.md` in `aerial-sample-apps`) | | OCUDU | SCTP | ASN.1 | dApp | Source. SCTP is off by default in libe3 (`LIBE3_ENABLE_SCTP`); libe3's SCTP tests skip on a host without the sctp kernel module. OCUDU's libe3-compatible mode is the 0.1.3 grammar ([#89](https://github.com/wineslab/libe3/issues/89)) | @@ -57,7 +58,8 @@ the same time, each with its own ports or endpoints and `ran_identifier`: the mu in CI runs two this way. A RAN that has peers on different links or encodings, such as Aerial (ZMQ, JSON) and OCUDU (SCTP, ASN.1), creates one agent for each. That wiring belongs to the RAN application, not to the library, and a peer uses the agent set up for it. Mixed links in one RAN have -not been run: SCTP is off in CI ([#81](https://github.com/wineslab/libe3/issues/81)). +not been run ([#81](https://github.com/wineslab/libe3/issues/81)). The POSIX and the io_uring link are +not mixed links in this sense: they are one wire, and a RAN of either serves a dApp of either. Defaults when nothing is set: ZeroMQ uses the setup, subscriber and publisher ports 9990, 9999 and 9991 and the IPC endpoints `/tmp/dapps/setup`, `dapp_socket` and `e3_socket`. The POSIX link uses one @@ -648,6 +650,7 @@ the same PR as the change. Newest first. | Change | PR | Kind | Who has to act | Status | |---|---|---|---|---| +| A third link layer, `E3LinkLayer::URING` (2): the POSIX link's endpoint, sockets and bytes driven through Linux io_uring, so a peer cannot tell it from POSIX. The SCTP associations of both socket links now set their retransmission timeout to 120 ms at first and 20 to 250 ms after, instead of Linux's 1 s smallest: a burst of large PDUs could close the peer's window and stall a send past the 500 ms send timeout until the RAN dropped the dApp ([#140](https://github.com/wineslab/libe3/issues/140)) | | Behavior (timing) | Nobody has to act: the bytes are unchanged and the timeout is each end's own. A peer sees faster window probes and retransmissions on SCTP. The dApp library and OAI's E3 agent offer `uring` only once their configuration parsers learn it | unmerged | | The POSIX link serves a dApp on one connection to one endpoint, as OCUDU does, on every transport. SCTP and IPC (now a sequenced-packet socket) carry one PDU per message with no length prefix; TCP prefixes every PDU with its length in both directions. SCTP stamps payload protocol id `0x00E3A001`, the RAN sends indications on stream 1 and everything else on stream 0, two streams are requested, send and receive buffers are set to 2 MiB and delayed SACK is off. A dApp is tied to its connection when it registers, and the RAN sends each PDU on the connection of the dApp it is for instead of to all. The IPC default is `/tmp/dapps/e3`; the port stays 9990. TCP and SCTP take IPv6 and a host in the endpoint, a list of peers and a cap on connections. New `E3Config` fields: `socket_buffer_bytes`, `sctp_delayed_sack`, `allowed_peers`, `maximum_associations` ([#104](https://github.com/wineslab/libe3/issues/104), [#105](https://github.com/wineslab/libe3/issues/105), [#106](https://github.com/wineslab/libe3/issues/106), [#107](https://github.com/wineslab/libe3/issues/107)) | | POSIX wire | Every POSIX peer, rebuilt together: libe3 RAN and dApp, OAI's E3 agent and the dApp library through their twin pull requests. Nothing of the old layout is kept: an old dApp meets a new RAN with a setup that times out or does not decode (the RAN logs why), and the reverse; an old IPC path no longer exists. `E3Connector` gains `send_to`, `last_setup_peer_token` and `PduClass`, and `create_connector` a `PosixOptions` argument. The patch OCUDU carries against libe3 is expected to be unnecessary for libe3's dApp role; this was not run against OCUDU's agent | unmerged | | `E3-ControlResponse` is the answer to a dApp control and to an xApp control. `E3-MessageAck` stays at index 10 of the PDU choice, marked DEPRECATED TO BE REMOVED, and libe3 sends none. Both controls get an optional `requestAck`, an extension addition written only when true. The response has OCUDU's fields and order without `fragmentIndex`, `fragmentCount` and `payloadBytes`, and `controlOutcome` of 1..32768 bytes. It takes index 11 of the PDU choice, as in OCUDU's grammar. The RAN answers a control it refuses itself (no setup, unknown function, unknown control id, no running Service Model) whether it asked or not, and a Service Model through `respond_to_control` only when the control asked. An xApp control that asks is answered by the dApp. A dApp reads only the responses that name it. JSON: type `controlResponse`, `controlOutcome` nested, `requestAck` only when true. Protobuf: `ControlResponse`, case 14. API: `respond_to_control`, `send_control_response`, `set_control_response_handler`, `e3_service_model_emit_control_response`, `e3_agent_set_control_response_handler`, `request_ack` on the two sends and `out_message_id` on `send_xapp_control` are new; the ack's helpers and handlers stay, deprecated. VERSION 0.5.0 ([#111](https://github.com/wineslab/libe3/issues/111)) | [#149](https://github.com/wineslab/libe3/pull/149) | ASN.1 wire, JSON and Protobuf, C++ API, C API, libe3py, behavior | libe3, OAI's E3 agent and the dApp library rebuilt together. Measured in a Linux container: libe3 0.4.3 refuses 0.5.0's response with `DECODE_FAILED`, so an unmatched dApp hears nothing where it heard an ack, and every earlier vector keeps its bytes. A Service Model that answered a control with the ack helpers should answer with `respond_to_control` or `e3_service_model_emit_control_response`: a message ack still goes out, and a libe3 dApp still reads it, but it is deprecated and no new code should send it. A dApp that sent controls and read nothing back sees negative responses for refusals now, as events, and no positive answer unless it sets `requestAck`. OCUDU's libe3-compatible mode re-vendors the grammar. Native OCUDU is unaffected and still not byte-compatible with libe3 for this message (outcome length, response code; measured, not run against its agent) | unmerged | | The interop page says that a libe3 RAN agent serves one link and one encoding, that agents with different configurations can run at the same time, and that a RAN with Aerial and OCUDU peers runs one for each ([#85](https://github.com/wineslab/libe3/issues/85)). It says that OCUDU can propose a third `E3-ResponseCode` on the page, and that libe3 is open to adding the extension marker with it ([#115](https://github.com/wineslab/libe3/issues/115)) | | Documentation | Nobody. libe3 does not change. A RAN application that serves both stacks creates the agents | unmerged | diff --git a/swig/README.md b/swig/README.md index b3065ed2..d19c915a 100644 --- a/swig/README.md +++ b/swig/README.md @@ -96,7 +96,7 @@ import libe3py as e3 cfg = e3.E3Config() cfg.role = e3.E3Role_DAPP -cfg.link_layer = e3.E3LinkLayer_ZMQ +cfg.link_layer = e3.E3LinkLayer_ZMQ # or E3LinkLayer_POSIX, or E3LinkLayer_URING (Linux, built with liburing) cfg.transport_layer = e3.E3TransportLayer_IPC cfg.encoding = e3.EncodingFormat_ASN1 # must match the RAN's E3Configuration cfg.dapp_name = "MyDApp" From d429af715262fcf0a725079e175bcbf755d1cfad Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Mon, 16 Nov 2026 15:49:51 -0500 Subject: [PATCH 7/7] docs: how to switch io_uring on for one group on a host that disables it Red Hat Enterprise Linux 9 and OpenShift's nodes ship kernel.io_uring_disabled=2, which refuses the io_uring link even to root. docs/connectors/uring.md gains a section with the least that has to be opened: set kernel.io_uring_group to a group of its own, set io_uring_disabled to 1, run the RAN or the dApp as a member of that group with no capability, and put both values back afterwards (from a trap, so that a failed run does not leave the node open), with the check that it took and that another user is still refused, and what changes in a pod (a privileged container is needed to write the sysctl, which is the host's and not namespaced, so it cannot be set from securityContext.sysctls). The README gets the same commands in a short section of its own, with a pointer to the page. This is what was done on the cluster for the pod measurements: the window was open for four minutes, the node read 2 and -1 again afterwards, a user outside the group was still refused while it was open, and one in the group created rings with no capability. Persisting the setting across reboots is described but was not tried. Assisted-by: Claude:claude-sonnet-5-5 Signed-off-by: Andrea Lacava --- README.md | 26 +++++++++++++++++++ docs/connectors/uring.md | 55 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d721d593..c5fa34c2 100644 --- a/README.md +++ b/README.md @@ -351,6 +351,32 @@ private: agent.register_sm(std::make_unique()); ``` +### The io_uring link + +Set `link_layer = E3LinkLayer::URING` (`--link uring` in the examples, `2` in the C API). It speaks the POSIX +wire, so it works against a RAN or a dApp on `posix`. It needs a libe3 built with liburing (`liburing-dev`), +Linux 5.11 or later, and a kernel that lets the process create a ring. Where it does not, `create_connector` +returns null and the log says which of these is missing; nothing falls back to another link. + +Red Hat Enterprise Linux 9 and OpenShift's nodes ship with io_uring switched off for everyone +(`kernel.io_uring_disabled=2`). The least that has to be opened is one group, for the length of the run, as root +on the host or in a privileged pod: + +```bash +GID=61234 # a group nothing else on the node uses +sysctl -w kernel.io_uring_group=$GID # set it first +sysctl -w kernel.io_uring_disabled=1 # 0 = everyone, 1 = root and that group, 2 = nobody + +setpriv --reuid=$UID --regid=$GID --groups=$GID --inh-caps=-all --bounding-set=-all --no-new-privs \ + ./example_simple_agent --link uring --transport tcp --encoding asn1 # a member of the group needs no capability + +sysctl -w kernel.io_uring_disabled=2 kernel.io_uring_group=-1 # close it again: the values you read before +``` + +The setting is the host kernel's, not the container's, and it applies to every container on the node. Ring +options, the checks that it took, a `trap` that closes it when a run fails, pods, and persisting it are in +[docs/connectors/uring.md](docs/connectors/uring.md#enabling-io_uring-on-a-host-that-disables-it). + ### Notes on Simulation / Local Testing The library exposes IPC endpoints by default (see `E3Config` defaults) so diff --git a/docs/connectors/uring.md b/docs/connectors/uring.md index 452c16e1..8a780431 100644 --- a/docs/connectors/uring.md +++ b/docs/connectors/uring.md @@ -204,6 +204,58 @@ which keeps the two links the same; `test_socket_link` reads the values back fro The tests that need io_uring say `URING NOT RUN` and exit 77 (CTest's "skipped") where it is refused, unless `LIBE3_REQUIRE_URING` is set: then they fail, so that CI cannot pass by skipping. +## Enabling io_uring on a host that disables it + +Red Hat Enterprise Linux 9 and the OpenShift nodes built on it ship `kernel.io_uring_disabled=2`, and +some other distributions do too. While it is 2, nothing can create a ring, not even root, and the +connector says so and refuses to start (`create_connector` returns null; the POSIX link needs none of this). +The setting belongs to the host kernel and is not namespaced: a pod cannot carry it in its +`securityContext.sysctls`, and whoever changes it changes it for every container on the node. So the +least exposure is to allow one group, for as long as the run takes: + +```sh +# 1. What the host has now. 0 = everyone, 1 = root and the members of io_uring_group, 2 = nobody. +cat /proc/sys/kernel/io_uring_disabled /proc/sys/kernel/io_uring_group # RHEL: 2 and -1 + +# 2. Open it for one group (as root on the host, or in a privileged pod). Set the group first. +GID=61234 # a group nothing else on the node uses +sysctl -w kernel.io_uring_group=$GID +sysctl -w kernel.io_uring_disabled=1 + +# 3. Run the RAN or the dApp as a member of that group. It needs no capability. +setpriv --reuid=$UID --regid=$GID --groups=$GID --inh-caps=-all --bounding-set=-all --no-new-privs \ + ./example_simple_agent --link uring --transport tcp --encoding asn1 + +# 4. Close it again: the values step 1 printed. +sysctl -w kernel.io_uring_disabled=2 kernel.io_uring_group=-1 +``` + +(`sysctl`, from procps, writes `/proc/sys`; `echo 1 > /proc/sys/kernel/io_uring_disabled` does the same.) +Do step 4 from a `trap` when the run is a script, so that a failed run does not leave the node open: + +```sh +orig_dis=$(cat /proc/sys/kernel/io_uring_disabled); orig_grp=$(cat /proc/sys/kernel/io_uring_group) +trap 'echo $orig_dis > /proc/sys/kernel/io_uring_disabled; echo $orig_grp > /proc/sys/kernel/io_uring_group' EXIT INT TERM HUP +``` + +To check that it took, and that it stayed closed to everyone else: run `LIBE3_REQUIRE_URING=1 +./test_link_availability` as a member of the group (it passes) and as another user (it says `URING NOT RUN` +and exits 77). After the run, read the two files from an unprivileged process on the node, a pod for +instance, to see the original values are back. + +- **In a pod.** Writing `/proc/sys` needs a privileged container, which on OpenShift needs the + `privileged` security context constraint and, where Pod Security is enforced, a namespace that allows + `privileged`. A privileged container is not under seccomp either, so there is nothing else to change. + A pod that is not privileged cannot write the sysctl, so the node has to be opened from outside it; its + own seccomp profile then has to allow `io_uring_setup`, `io_uring_enter` and `io_uring_register` + (`Unconfined`, or a `Localhost` profile that does). +- **To keep it open across reboots**, put the two lines in a file under `/etc/sysctl.d/`, or in whatever + your platform uses for node settings (on OpenShift a MachineConfig); not tried here. That makes the + allowance permanent for the group, so keep the group narrow. +- **What it costs in exposure.** io_uring is a large part of the kernel's attack surface, which is why it + is off by default; the group form limits it to the processes you put in the group (and to root, as the + kernel documents `1`). Everything else on the node stays refused. + ## Ring options Read from the environment when the connector is made (`E3Interface::init()`, or each restart of a @@ -230,7 +282,8 @@ dApp). A value out of range is ignored with a warning. as well; one per connector, so two for a RAN and a dApp in one process. - In a container: allow io_uring in the seccomp profile. In Kubernetes and OpenShift that is a `seccompProfile` of type `Localhost` (or `Unconfined`), and on a node whose kernel ships - `kernel.io_uring_disabled=2`, a node-level sysctl change. + `kernel.io_uring_disabled=2`, a node-level sysctl change (see "Enabling io_uring on a host that + disables it" above for the least that has to be opened). - An SM control handler that makes socket calls with a timeout on the RAN's inbound thread should retry on `EINTR` (decision 6), or the deployment should use the kernel submission thread.