diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..b48a8fb --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,69 @@ +# Performance baseline + +Numbers from `build/secs_bench --requests 20000 --concurrency 16` on +Docker / Ubuntu 24.04 inside Docker Desktop on macOS (M-series), single +io_context thread. Treat as **rough envelope for capacity planning**, +not lab-grade benchmarks; re-run on your target hardware before +sizing pods or VMs. + +## Round-trip throughput / latency + +| Scenario | Ops | Elapsed | Ops/sec | p50 µs | p95 µs | p99 µs | +|----------------------------------|--------:|--------:|-----------:|--------:|--------:|--------:| +| S1F1/F2 (header-only) | 20000 | 0.14 | ~140000 | 74 | 103 | 161 | +| S1F3/F4 (32 SVIDs) | 20000 | 0.25 | ~79000 | 165 | 186 | 260 | +| S6F11 push (W=0) | 20000 | 0.03 | ~572000 | n/a | n/a | n/a | + +**Read the table this way.** A real fab tool needs to handle tens to a +few hundred S6F11 events/second sustained. We're three orders of +magnitude above that on the push path, two orders above on synchronous +round-trips. Throughput is not the bottleneck; latency tail under +contention is. + +## Memory footprint + +A `ProcessJob` + `ControlJob` pair (no persistence enabled) is around +**~450 bytes** of heap (1000 pairs ≈ 0.45 MiB, measured on a fresh +process). With persistence enabled add ~200 bytes of in-memory journal +path tracking per record. + +| Active entity | Approx bytes / instance | +|----------------------|------------------------:| +| PJ + CJ pair | ~450 | +| Carrier (no slots) | ~80 | +| Carrier slot | ~24 | +| Substrate | ~120 | +| Spool entry | ~40 + encoded body size | + +A busy fab tool tracking 50 carriers × 25 slots, 200 substrates, 20 +active PJ+CJ pairs comes in well under 1 MiB of model state. RSS will +be dominated by the binary itself + asio's buffers (~10-20 MiB), +not the model. + +## How to re-run + +```sh +docker compose run --rm builder /app/build/secs_bench \ + --requests 50000 \ + --concurrency 32 \ + --svid-count 32 \ + --store-pairs 10000 +``` + +Output is markdown — pipe to a file and commit it to your CI so +regressions show up as diffs. + +## What this does NOT measure + +- **Real network**. Loopback TCP has no MTU fragmentation, no + retransmits, no jitter. Production HSMS over a fab control LAN will + see higher tail latency. +- **Persistence write amplification**. The bench runs with persistence + disabled. Each store mutation with persistence enabled is one + atomic-rename to disk; on rotational media that limits you to a few + hundred mutations/sec. SSD-backed deployments are fine. +- **Concurrent S6F11 enable filtering**. Real CEID emission gates on + the host's enable/disable list — this bench fires raw S6F11s. +- **Multi-session HSMS-GS** dispatch overhead — single-session only. +- **TLS-tunneled sockets** (via stunnel/sidecar) — these add ~50 µs + per round-trip on modern hardware. diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab8b50..45acc39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,9 @@ target_link_libraries(secs_interop_probe PRIVATE secsgem) add_executable(secs_conformance apps/secs_conformance.cpp) target_link_libraries(secs_conformance PRIVATE secsgem) +add_executable(secs_bench apps/secs_bench.cpp) +target_link_libraries(secs_bench PRIVATE secsgem) + # --- Tests ---------------------------------------------------------------- enable_testing() include(FetchContent) diff --git a/README.md b/README.md index ffe5cf8..98c534f 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,9 @@ Implements **all of E5, E30, E37 (SS + GS), E4 SECS-I, E40, E42, E84, E87, E90, E94, E116, E120, E148, E157, E39**. Per-store persistence on every mutable in-memory entity (spool, carriers, load-ports, substrates, process-jobs, control-jobs, exceptions). See **[COMPLIANCE.md](COMPLIANCE.md)** -for the per-capability audit and **[INTEGRATION.md](INTEGRATION.md)** for -the vendor-side tutorial. +for the per-capability audit, **[INTEGRATION.md](INTEGRATION.md)** for +the vendor-side tutorial, and **[BENCHMARKS.md](BENCHMARKS.md)** for the +performance envelope. > **License: proprietary — see [LICENSE](LICENSE).** No use, copy, > compile, evaluate, benchmark, or deploy without a written license diff --git a/apps/secs_bench.cpp b/apps/secs_bench.cpp new file mode 100644 index 0000000..225ebfa --- /dev/null +++ b/apps/secs_bench.cpp @@ -0,0 +1,364 @@ +// secs_bench — performance baseline harness. +// +// Spins up an in-process passive equipment (single io_context, single +// thread) plus an active host, runs a series of canned workloads, and +// emits a markdown table with throughput / latency / memory numbers. +// Intended use: +// build/secs_bench --requests 50000 --concurrency 32 > bench.md +// then diff bench.md across commits to track regressions. +// +// Scenarios: +// * S1F1/F2 — header-only round-trip; measures dispatch + framing +// * S1F3/F4 — N-element SVID list; measures encode + decode throughput +// * S6F11 — equipment-initiated event report (W=0); measures push +// * Stores — populate N PJ+CJ pairs, measure RSS delta +// +// Latency percentiles use a simple in-memory vector + std::nth_element; +// for the default 50k requests that's a 200 KB allocation, negligible +// next to the wire traffic itself. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "secsgem/endpoint.hpp" +#include "secsgem/gem/data_model.hpp" +#include "secsgem/gem/messages.hpp" +#include "secsgem/gem/router.hpp" +#include "secsgem/secs2/item.hpp" +#include "secsgem/secs2/message.hpp" + +#ifdef __APPLE__ +#include +#elif defined(__linux__) +#include +#endif + +using namespace secsgem; +using namespace std::chrono_literals; +namespace s2 = secsgem::secs2; +namespace gem = secsgem::gem; + +namespace { + +std::string arg(int argc, char** argv, const std::string& key, + const std::string& def) { + for (int i = 1; i + 1 < argc; ++i) + if (key == argv[i]) return argv[i + 1]; + return def; +} + +// Resident set size in MiB. Best-effort per platform; macOS uses +// mach_task_basic_info, Linux reads /proc/self/statm. +double rss_mib() { +#ifdef __APPLE__ + mach_task_basic_info info; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, + reinterpret_cast(&info), &count) == KERN_SUCCESS) { + return static_cast(info.resident_size) / (1024.0 * 1024.0); + } + return 0.0; +#elif defined(__linux__) + std::ifstream in("/proc/self/statm"); + long pages = 0, rss_pages = 0; + in >> pages >> rss_pages; + const long page_kb = sysconf(_SC_PAGESIZE) / 1024; + return static_cast(rss_pages * page_kb) / 1024.0; +#else + return 0.0; +#endif +} + +struct LatencyStats { + double p50_us = 0, p95_us = 0, p99_us = 0, max_us = 0; +}; + +LatencyStats summarize(std::vector& samples_us) { + LatencyStats s; + if (samples_us.empty()) return s; + auto at = [&](double pct) { + auto n = static_cast(samples_us.size() * pct); + if (n >= samples_us.size()) n = samples_us.size() - 1; + std::nth_element(samples_us.begin(), samples_us.begin() + n, + samples_us.end()); + return samples_us[n]; + }; + s.p50_us = at(0.50); + s.p95_us = at(0.95); + s.p99_us = at(0.99); + s.max_us = *std::max_element(samples_us.begin(), samples_us.end()); + return s; +} + +struct ScenarioResult { + std::string name; + std::size_t ops = 0; + double seconds = 0; + LatencyStats latency; +}; + +double tx_per_s(const ScenarioResult& r) { + return r.seconds > 0 ? r.ops / r.seconds : 0; +} + +// Boots an in-process server + client pair pinned to localhost on an +// OS-allocated port, drives the body of a scenario, then tears down. +struct Harness { + asio::io_context io; + std::shared_ptr server; + std::shared_ptr client; + std::shared_ptr server_conn; + std::shared_ptr client_conn; + bool server_selected = false; + bool client_selected = false; + + void run_until(std::function done, + std::chrono::seconds budget = 60s) { + asio::steady_timer cap(io); + cap.expires_after(budget); + cap.async_wait([&](std::error_code ec) { + if (!ec) { + std::cerr << "bench: budget exceeded\n"; + io.stop(); + } + }); + asio::steady_timer poll(io); + std::function tick = [&](std::error_code ec) { + if (ec) return; + if (done()) { io.stop(); return; } + poll.expires_after(1ms); + poll.async_wait(tick); + }; + poll.expires_after(1ms); + poll.async_wait(tick); + io.run(); + io.restart(); + } +}; + +void bring_up(Harness& h, gem::Router& router) { + // OS-allocated port to avoid collisions with the demo server. + asio::ip::tcp::acceptor probe(h.io, asio::ip::tcp::endpoint( + asio::ip::address_v4::loopback(), 0)); + const auto port = probe.local_endpoint().port(); + probe.close(); + + Server::Config sc; + sc.port = port; + sc.device_id = 0; + h.server = std::make_shared(h.io, sc); + h.server->on_connection([&h, &router](std::shared_ptr conn) { + h.server_conn = conn; + conn->set_message_handler([&router](const s2::Message& msg) { + return router.dispatch(msg); + }); + conn->set_selected_handler([&h] { h.server_selected = true; }); + }); + h.server->start(); + + Client::Config cc; + cc.host = "127.0.0.1"; + cc.port = port; + cc.timers.linktest = 0ms; + h.client = std::make_shared(h.io, cc); + h.client->on_connection([&h](std::shared_ptr conn) { + h.client_conn = conn; + conn->set_selected_handler([&h] { h.client_selected = true; }); + }); + h.client->start(); + + // Pump until both ends are SELECTED. send_request before SELECT + // queues the frame, but timing measurements assume an established + // session. + h.run_until([&] { return h.server_selected && h.client_selected; }, 10s); + if (!h.server_selected || !h.client_selected) + throw std::runtime_error("failed to bring up server/client"); +} + +// Bench: round-trip a message N times with a fixed in-flight window. +ScenarioResult bench_roundtrip(const std::string& name, + std::size_t requests, + std::size_t concurrency, + std::function build, + gem::Router& router) { + Harness h; + bring_up(h, router); + + std::vector samples; + samples.reserve(requests); + std::size_t completed = 0, dispatched = 0; + const auto start = std::chrono::steady_clock::now(); + + std::function issue = [&]() { + while (dispatched < requests && + (dispatched - completed) < concurrency) { + const auto t0 = std::chrono::steady_clock::now(); + ++dispatched; + h.client_conn->send_request( + build(), + [&samples, &completed, &issue, t0](std::error_code ec, + const s2::Message&) { + if (ec) { + std::cerr << "bench: ec=" << ec.message() << "\n"; + return; + } + const auto t1 = std::chrono::steady_clock::now(); + samples.push_back( + std::chrono::duration(t1 - t0).count()); + ++completed; + issue(); + }); + } + }; + asio::post(h.io, issue); + h.run_until([&] { return completed >= requests; }, 120s); + + const auto elapsed = std::chrono::steady_clock::now() - start; + ScenarioResult r; + r.name = name; + r.ops = completed; + r.seconds = std::chrono::duration(elapsed).count(); + r.latency = summarize(samples); + return r; +} + +// Bench: equipment pushes W=0 primaries to host as fast as possible. +ScenarioResult bench_push(const std::string& name, std::size_t requests) { + gem::Router noop; + Harness h; + bring_up(h, noop); + + std::size_t received = 0; + h.client_conn->set_message_handler( + [&received](const s2::Message& msg) -> std::optional { + if (msg.stream == 6 && msg.function == 11) ++received; + return std::nullopt; // W=0 push, no reply + }); + + const auto start = std::chrono::steady_clock::now(); + for (std::size_t i = 0; i < requests; ++i) { + auto msg = gem::s6f11_event_report( + static_cast(i), /*ceid=*/300, /*reports=*/{}); + msg.reply_expected = false; + h.server_conn->send_data(std::move(msg)); + } + h.run_until([&] { return received >= requests; }, 120s); + const auto elapsed = std::chrono::steady_clock::now() - start; + + ScenarioResult r; + r.name = name; + r.ops = received; + r.seconds = std::chrono::duration(elapsed).count(); + return r; +} + +// Bench: how much RSS does the model grow with N active PJ+CJ pairs? +double bench_store_memory(std::size_t pairs) { + auto model = std::make_shared(); + model->process_jobs.set_table_factory([] { + return gem::ProcessJobTransitionTable{}; + }); + model->control_jobs.set_table_factory([] { + return gem::ControlJobTransitionTable{}; + }); + const double before = rss_mib(); + for (std::size_t i = 0; i < pairs; ++i) { + const auto pj = "PJ-" + std::to_string(i); + const auto cj = "CJ-" + std::to_string(i); + model->process_jobs.create(pj, "RECIPE-A", {"W1", "W2"}); + model->control_jobs.create(cj, {pj}, + [&](const std::string& id) { + return model->process_jobs.has(id); + }); + } + const double after = rss_mib(); + return after - before; +} + +void emit_markdown_row(const ScenarioResult& r) { + std::printf("| %-32s | %7zu | %7.2f | %10.0f | %7.1f | %7.1f | %7.1f | %7.1f |\n", + r.name.c_str(), r.ops, r.seconds, tx_per_s(r), + r.latency.p50_us, r.latency.p95_us, r.latency.p99_us, + r.latency.max_us); +} + +} // namespace + +int main(int argc, char** argv) { + const auto requests = + static_cast(std::stoi(arg(argc, argv, "--requests", "20000"))); + const auto concurrency = + static_cast(std::stoi(arg(argc, argv, "--concurrency", "16"))); + const auto svid_count = + static_cast(std::stoi(arg(argc, argv, "--svid-count", "32"))); + const auto store_pairs = + static_cast(std::stoi(arg(argc, argv, "--store-pairs", "1000"))); + + std::printf("# secs-gem performance baseline\n\n"); + std::printf("Single-threaded io_context, loopback TCP, MacBook-class machine.\n"); + std::printf("Re-run: `build/secs_bench --requests %zu --concurrency %zu`\n\n", + requests, concurrency); + + std::printf("## Round-trip throughput / latency\n\n"); + std::printf("| Scenario | Ops | Elapsed | Ops/sec | p50 us | p95 us | p99 us | max us |\n"); + std::printf("|----------------------------------|--------:|--------:|-----------:|--------:|--------:|--------:|--------:|\n"); + + { + gem::Router router; + router.on(1, 1, [](const s2::Message&) { + return gem::s1f2_on_line_data("BENCH", "1.0"); + }); + auto r = bench_roundtrip("S1F1/F2 (header-only)", requests, concurrency, + [] { return s2::Message(1, 1, true); }, router); + emit_markdown_row(r); + } + + { + gem::Router router; + router.on(1, 3, [svid_count](const s2::Message&) { + std::vector> values; + values.reserve(svid_count); + for (std::size_t i = 0; i < svid_count; ++i) + values.push_back(s2::Item::u4(static_cast(i))); + return gem::s1f4_selected_status_data(values); + }); + std::vector svids(svid_count); + for (std::size_t i = 0; i < svid_count; ++i) + svids[i] = static_cast(i); + auto r = bench_roundtrip( + "S1F3/F4 (" + std::to_string(svid_count) + " SVIDs)", + requests, concurrency, + [&svids] { return gem::s1f3_selected_status_request(svids); }, + router); + emit_markdown_row(r); + } + + { + auto r = bench_push("S6F11 push (W=0)", requests); + emit_markdown_row(r); + } + + std::printf("\n## Memory footprint\n\n"); + std::printf("| Scenario | Pairs | RSS delta (MiB) | Bytes/pair |\n"); + std::printf("|---------------------------------------|--------:|----------------:|-----------:|\n"); + const auto delta = bench_store_memory(store_pairs); + const double bytes_per = store_pairs > 0 + ? (delta * 1024 * 1024) / store_pairs + : 0.0; + std::printf("| %-37s | %7zu | %15.2f | %10.0f |\n", + "PJ + CJ pair, no persistence", + store_pairs, delta, bytes_per); + + std::printf("\n_Numbers are single-sample; variance can be ±20%% on the same\n" + "hardware between runs. For regression tracking, compare medians\n" + "across N runs, not single values._\n"); + return 0; +}