secs-gem/tests/test_robustness_fuzz.cpp

// Randomized robustness / property test.
//
// Spins up a fresh EquipmentDataModel pre-loaded with realistic
// SVIDs/ECIDs/CEIDs/alarms/recipes and drives a long sequence of
// random tool operations against it.  Each action is "adjusted" — the
// fuzz picks a random verb but checks state-machine legality before
// applying, so a Pause is only fired on a Processing PJ, a Recover
// only on a Posted exception, etc.  Illegal-by-state attempts are
// no-ops counted but not faulted; they exercise the FSM guards.
//
// After every N ticks the fuzz checks a battery of invariants:
//   * referential integrity (CJ prjobids exist in PJ store, defined
//     reports' VIDs exist, linked CEIDs are registered, etc.)
//   * resource accounting (size() matches enumeration count)
//   * no FSM in NoState (sentinel must never leak as a live state)
//   * EPT bucket accumulator monotonically non-decreasing
//   * active alarms ⊆ declared alarms
//   * spool depth matches enumerated entries
//
// At configurable intervals it does a full persistence round-trip:
//   write every store to a tempdir, load into a fresh model, diff
//   the in-memory state.  This is the strongest end-to-end check we
//   have for "the equipment can survive a power loss" claim.
//
// Reproducibility: every test case uses a fixed seed.  On failure the
// last ~20 applied actions are printed so the sequence can be
// replayed (or pasted into a targeted unit test).
//
// Coverage scaling: 4 seeds × 2000 ticks for the fast CI suite.  Set
// SECSGEM_ROBUSTNESS_SOAK=1 to run an extra 100k-tick test that
// exercises the long-running pathological-distribution case.

#include <doctest/doctest.h>

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <deque>
#include <filesystem>
#include <functional>
#include <random>
#include <set>
#include <sstream>
#include <string>
#include <vector>

#include "secsgem/gem/data_model.hpp"
#include "secsgem/secs2/item.hpp"

using namespace secsgem;
using namespace secsgem::gem;
namespace fs = std::filesystem;

namespace {

// ---------------------------------------------------------------------------
// Scratch directory helpers — used both for persistence round-trips and
// for the soak variant's append-only journal.
// ---------------------------------------------------------------------------

fs::path scratch_root(uint64_t seed) {
  std::random_device rd;
  auto p = fs::temp_directory_path() /
           ("secsgem-fuzz-" + std::to_string(seed) + "-" + std::to_string(rd()));
  fs::remove_all(p);
  fs::create_directories(p);
  return p;
}

// ---------------------------------------------------------------------------
// World: an EquipmentDataModel pre-loaded with a representative dictionary
// + ID-tracking + an RNG.  Every Action operates on the World.
// ---------------------------------------------------------------------------

struct ActionEvent {
  uint64_t tick;
  std::string what;
};

struct World {
  EquipmentDataModel model;
  std::mt19937_64 rng;
  fs::path journal_dir;  // persistence enabled at startup; dir owned by World

  // Catalog: created at startup, never mutated by the fuzz.
  std::vector<uint32_t> svid_ids;
  std::vector<uint32_t> dvid_ids;
  std::vector<uint32_t> ecid_ids;
  std::vector<uint32_t> ceid_ids;
  std::vector<uint32_t> alarm_ids;
  std::vector<std::string> recipe_ids;

  // Live IDs the fuzz adds/removes over time.
  std::vector<std::string> pjs;
  std::vector<std::string> cjs;
  std::vector<std::string> carriers;
  std::vector<std::string> substrates;
  std::vector<uint32_t>    exceptions;
  std::vector<std::string> modules;
  std::vector<uint32_t>    report_ids;

  // Bookkeeping for invariants.
  uint64_t prev_ept_total_ms = 0;
  std::deque<ActionEvent> recent_actions;  // for failure diagnostics
  uint64_t tick = 0;

  explicit World(uint64_t seed) : rng(seed) {
    journal_dir = scratch_root(seed);
    // Enable persistence BEFORE populate so even initial state participates.
    // Order matters: the per-store dirs need to exist before mutations.
    model.process_jobs.set_table_factory(
        [] { return ProcessJobTransitionTable::default_table(); });
    model.control_jobs.set_table_factory(
        [] { return ControlJobTransitionTable::default_table(); });
    model.process_jobs.enable_persistence(journal_dir / "pj");
    model.control_jobs.enable_persistence(journal_dir / "cj");
    model.carriers.enable_persistence(journal_dir / "car");
    model.load_ports.enable_persistence(journal_dir / "lp");
    model.substrates.enable_persistence(journal_dir / "sub");
    model.exceptions.enable_persistence(journal_dir / "ex");
    model.spool.enable_persistence(journal_dir / "spool");
    populate_catalog_();
  }
  ~World() {
    std::error_code ec;
    fs::remove_all(journal_dir, ec);
  }

  // -------------------------------------------------------------------------
  // Catalog setup — a representative GEM 300 tool's data dictionary.
  // -------------------------------------------------------------------------
  void populate_catalog_() {
    // SVIDs.
    for (uint32_t id : {1u, 2u, 3u, 100u, 101u, 102u}) {
      model.svids.add({id, "SV-" + std::to_string(id), "u",
                        secs2::Item::u4(0)});
      svid_ids.push_back(id);
    }
    // DVIDs.
    for (uint32_t id : {200u, 201u, 202u}) {
      model.dvids.add({id, "DV-" + std::to_string(id), "u",
                        secs2::Item::f4(0.0f)});
      dvid_ids.push_back(id);
    }
    // ECIDs.
    for (uint32_t id : {300u, 301u, 302u}) {
      const auto v = secs2::Item::u4(1);
      model.ecids.add({id, "EC-" + std::to_string(id), "u",
                        v, v, "0", "1000"});
      ecid_ids.push_back(id);
    }
    // CEIDs.
    for (uint32_t id : {1000u, 1001u, 1002u, 1003u}) {
      model.events.register_event({id, "CE-" + std::to_string(id)});
      ceid_ids.push_back(id);
    }
    // Alarms.
    for (uint32_t id : {1u, 2u, 3u}) {
      model.alarms.add({id, "AL-" + std::to_string(id), /*cat=*/0x04});
      alarm_ids.push_back(id);
    }
    // Recipes.
    for (const auto& id : {std::string("R-A"), std::string("R-B")}) {
      model.recipes.add(id, "BODY");
      recipe_ids.push_back(id);
    }
    // Modules: create three named.
    for (const auto& id : {std::string("M-1"), std::string("M-2"),
                            std::string("M-3")}) {
      model.modules.create(id);
      modules.push_back(id);
    }
    // FSM table factories already wired in the constructor before
    // persistence-enable so replay finds matching factories.
  }

  void log_action(const std::string& what) {
    recent_actions.push_back({tick, what});
    if (recent_actions.size() > 20) recent_actions.pop_front();
  }

  std::string trace() const {
    std::ostringstream os;
    os << "\nrecent actions (most recent last):\n";
    for (const auto& e : recent_actions)
      os << "  t=" << e.tick << "  " << e.what << "\n";
    return os.str();
  }

  template <typename Vec>
  const typename Vec::value_type* pick(const Vec& v) {
    if (v.empty()) return nullptr;
    return &v[std::uniform_int_distribution<std::size_t>(0, v.size() - 1)(rng)];
  }

  bool roll(double p) {
    return std::uniform_real_distribution<>(0.0, 1.0)(rng) < p;
  }
};

// ---------------------------------------------------------------------------
// Actions.  Each returns true if the operation landed (state actually
// changed), false if it was a no-op (no legal target / illegal in current
// state).  Even no-ops count as exercising the FSM guards.
// ---------------------------------------------------------------------------

bool act_pj_create(World& w) {
  auto* ppid = w.pick(w.recipe_ids);
  if (!ppid) return false;
  const auto id = "PJ-" + std::to_string(w.tick) + "-" +
                  std::to_string(w.rng() % 0xFFFF);
  auto r = w.model.process_jobs.create(id, *ppid, {"W-1"});
  if (r != ProcessJobStore::CreateResult::Created) return false;
  w.pjs.push_back(id);
  w.log_action("pj_create " + id + " ppid=" + *ppid);
  return true;
}

bool act_pj_event(World& w) {
  auto* id = w.pick(w.pjs);
  if (!id) return false;
  static const ProcessJobEvent events[] = {
      ProcessJobEvent::Select,         ProcessJobEvent::SetupComplete,
      ProcessJobEvent::Start,          ProcessJobEvent::Pause,
      ProcessJobEvent::Resume,         ProcessJobEvent::Stop,
      ProcessJobEvent::Abort,          ProcessJobEvent::HeadOfQueue,
      ProcessJobEvent::ProcessComplete,ProcessJobEvent::AbortComplete};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  const bool ok = w.model.process_jobs.fire_internal(*id, ev);
  if (ok) w.log_action("pj_event " + *id + " " + process_job_event_name(ev));
  return ok;
}

bool act_pj_dequeue(World& w) {
  if (w.pjs.empty()) return false;
  // Build the set of PJ ids currently bound to any active CJ — those
  // are NOT dequeueable per E94 (the application gates this; the store
  // would happily dequeue a CJ-bound PJ and leave a dangling reference,
  // which the I-2 invariant correctly flags).  Mirror the gate here.
  std::set<std::string> bound;
  for (const auto& cj_id : w.cjs) {
    auto* cj = w.model.control_jobs.get(cj_id);
    if (!cj) continue;
    for (const auto& pj : cj->prjobids) bound.insert(pj);
  }
  std::vector<std::size_t> candidates;
  for (std::size_t i = 0; i < w.pjs.size(); ++i)
    if (!bound.count(w.pjs[i])) candidates.push_back(i);
  if (candidates.empty()) return false;

  const auto idx = candidates[w.rng() % candidates.size()];
  const auto id = w.pjs[idx];
  auto ack = w.model.process_jobs.dequeue(id);
  if (ack != HostCmdAck::Accept) return false;
  w.pjs.erase(w.pjs.begin() + idx);
  w.log_action("pj_dequeue " + id);
  return true;
}

bool act_cj_create(World& w) {
  if (w.pjs.empty()) return false;
  std::vector<std::string> sample;
  const auto n = std::min<std::size_t>(1 + (w.rng() % 3), w.pjs.size());
  std::set<std::size_t> picked;
  while (sample.size() < n) {
    auto idx = w.rng() % w.pjs.size();
    if (picked.insert(idx).second) sample.push_back(w.pjs[idx]);
  }
  const auto id = "CJ-" + std::to_string(w.tick);
  auto r = w.model.control_jobs.create(
      id, sample, [&](const std::string& pj) { return w.model.process_jobs.has(pj); });
  if (r != ControlJobStore::CreateResult::Created) return false;
  w.cjs.push_back(id);
  w.log_action("cj_create " + id + " pjs=" + std::to_string(sample.size()));
  return true;
}

bool act_cj_event(World& w) {
  auto* id = w.pick(w.cjs);
  if (!id) return false;
  static const ControlJobEvent events[] = {
      ControlJobEvent::Select,        ControlJobEvent::SetupComplete,
      ControlJobEvent::Start,         ControlJobEvent::Pause,
      ControlJobEvent::Resume,        ControlJobEvent::Stop,
      ControlJobEvent::Abort,         ControlJobEvent::AllJobsComplete,
      ControlJobEvent::AbortComplete};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  const bool ok = w.model.control_jobs.fire_internal(*id, ev);
  if (ok) w.log_action("cj_event " + *id + " " + control_job_event_name(ev));
  return ok;
}

bool act_cj_delete(World& w) {
  if (w.cjs.empty()) return false;
  auto idx = w.rng() % w.cjs.size();
  const auto id = w.cjs[idx];
  if (!w.model.control_jobs.remove(id)) return false;
  w.cjs.erase(w.cjs.begin() + idx);
  w.log_action("cj_delete " + id);
  return true;
}

bool act_carrier_create(World& w) {
  const auto id = "CAR-" + std::to_string(w.tick);
  const uint8_t port = 1 + (w.rng() % 4);
  // 25-slot FOUP-shaped carrier.
  auto r = w.model.carriers.create(id, port, /*capacity=*/25);
  if (r != CarrierStore::CreateResult::Created) return false;
  w.carriers.push_back(id);
  w.log_action("carrier_create " + id + " port=" + std::to_string(port));
  return true;
}

bool act_carrier_id_event(World& w) {
  auto* id = w.pick(w.carriers);
  if (!id) return false;
  static const CarrierIDEvent events[] = {
      CarrierIDEvent::Read,
      CarrierIDEvent::ProceedWithCarrier,
      CarrierIDEvent::CancelCarrier,
      CarrierIDEvent::Bind};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  const bool ok = w.model.carriers.fire_id_event(*id, ev);
  if (ok) w.log_action("carrier_id " + *id);
  return ok;
}

bool act_carrier_slot_event(World& w) {
  auto* id = w.pick(w.carriers);
  if (!id) return false;
  static const SlotMapEvent events[] = {
      SlotMapEvent::Read, SlotMapEvent::Confirm, SlotMapEvent::Mismatch};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  const bool ok = w.model.carriers.fire_slot_map_event(*id, ev);
  if (ok) w.log_action("carrier_slot " + *id);
  return ok;
}

bool act_substrate_create(World& w) {
  auto* carrier = w.pick(w.carriers);
  if (!carrier) return false;
  const auto id = "WFR-" + std::to_string(w.tick);
  const uint8_t slot = 1 + (w.rng() % 25);
  auto r = w.model.substrates.create(id, *carrier, slot);
  if (r != SubstrateStore::CreateResult::Created) return false;
  w.substrates.push_back(id);
  w.log_action("substrate_create " + id);
  return true;
}

bool act_substrate_location(World& w) {
  auto* id = w.pick(w.substrates);
  if (!id) return false;
  static const SubstrateEvent events[] = {
      SubstrateEvent::Acquire, SubstrateEvent::Release,
      SubstrateEvent::Return};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  const bool ok = w.model.substrates.fire_location_event(*id, ev);
  if (ok) w.log_action("substrate_loc " + *id);
  return ok;
}

bool act_substrate_processing(World& w) {
  auto* id = w.pick(w.substrates);
  if (!id) return false;
  static const SubstrateProcessingEvent events[] = {
      SubstrateProcessingEvent::StartProcessing,
      SubstrateProcessingEvent::EndProcessing,
      SubstrateProcessingEvent::Abort,
      SubstrateProcessingEvent::Stop,
      SubstrateProcessingEvent::Reject,
      SubstrateProcessingEvent::ReportLost,
      SubstrateProcessingEvent::Skip};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  const bool ok = w.model.substrates.fire_processing_event(*id, ev);
  if (ok) w.log_action("substrate_proc " + *id);
  return ok;
}

bool act_alarm_set(World& w) {
  auto* id = w.pick(w.alarm_ids);
  if (!id) return false;
  const bool ok = w.model.alarms.set_active(*id).has_value();
  if (ok) w.log_action("alarm_set " + std::to_string(*id));
  return ok;
}

bool act_alarm_clear(World& w) {
  auto* id = w.pick(w.alarm_ids);
  if (!id) return false;
  if (!w.model.alarms.active(*id)) return false;
  const bool ok = w.model.alarms.clear_active(*id).has_value();
  if (ok) w.log_action("alarm_clear " + std::to_string(*id));
  return ok;
}

bool act_alarm_enable_toggle(World& w) {
  auto* id = w.pick(w.alarm_ids);
  if (!id) return false;
  const bool now = w.model.alarms.enabled(*id);
  w.model.alarms.set_enabled(*id, !now);
  w.log_action("alarm_enable " + std::to_string(*id));
  return true;
}

bool act_svid_update(World& w) {
  auto* id = w.pick(w.svid_ids);
  if (!id) return false;
  w.model.svids.set_value(*id, secs2::Item::u4(
      static_cast<uint32_t>(w.rng() & 0xFFFFFFFFu)));
  return true;  // always succeeds; don't pollute action log
}

bool act_define_report(World& w) {
  if (w.svid_ids.empty()) return false;
  const uint32_t rpt = 5000 + static_cast<uint32_t>(w.tick % 100);
  std::vector<uint32_t> vids;
  const auto n = 1 + (w.rng() % 3);
  for (std::size_t i = 0; i < n && i < w.svid_ids.size(); ++i)
    vids.push_back(w.svid_ids[w.rng() % w.svid_ids.size()]);
  auto ack = w.model.define_reports({{rpt, vids}});
  if (ack == DefineReportAck::Accept) {
    if (std::find(w.report_ids.begin(), w.report_ids.end(), rpt) ==
        w.report_ids.end())
      w.report_ids.push_back(rpt);
    w.log_action("define_report " + std::to_string(rpt));
    return true;
  }
  return false;
}

bool act_link_event(World& w) {
  if (w.report_ids.empty()) return false;
  auto* ceid = w.pick(w.ceid_ids);
  if (!ceid) return false;
  std::vector<uint32_t> rpts;
  rpts.push_back(w.report_ids[w.rng() % w.report_ids.size()]);
  auto ack = w.model.link_event_reports({{*ceid, rpts}});
  if (ack == LinkEventAck::Accept) {
    w.log_action("link_event ceid=" + std::to_string(*ceid));
    return true;
  }
  return false;
}

bool act_enable_event(World& w) {
  auto* ceid = w.pick(w.ceid_ids);
  if (!ceid) return false;
  const bool enable = w.roll(0.7);
  auto ack = w.model.enable_events(enable, {*ceid});
  if (ack == EnableEventAck::Accept) {
    w.log_action(std::string(enable ? "enable" : "disable") +
                  "_event " + std::to_string(*ceid));
    return true;
  }
  return false;
}

bool act_exception_post(World& w) {
  const uint32_t exid = 9000 + static_cast<uint32_t>(w.tick);
  auto r = w.model.exceptions.post(exid, "VACUUM", "lost vacuum",
                                     {"PURGE", "RECOVER"});
  if (r != ExceptionStore::PostResult::Posted) return false;
  w.exceptions.push_back(exid);
  w.log_action("exception_post " + std::to_string(exid));
  return true;
}

bool act_exception_recover(World& w) {
  if (w.exceptions.empty()) return false;
  const uint32_t exid = w.exceptions[w.rng() % w.exceptions.size()];
  const auto* ex = w.model.exceptions.get(exid);
  if (!ex) return false;
  if (ex->fsm->state() != ExceptionState::Posted &&
      ex->fsm->state() != ExceptionState::RecoverFailed)
    return false;
  auto ack = w.model.exceptions.on_recover(exid, "PURGE");
  if (ack != AlarmAck::Accept) return false;
  w.log_action("exception_recover " + std::to_string(exid));
  return true;
}

bool act_exception_complete(World& w) {
  if (w.exceptions.empty()) return false;
  const uint32_t exid = w.exceptions[w.rng() % w.exceptions.size()];
  const auto* ex = w.model.exceptions.get(exid);
  if (!ex) return false;
  if (ex->fsm->state() != ExceptionState::Recovering) return false;
  const auto event = w.roll(0.7) ? ExceptionEvent::RecoveryComplete
                                  : ExceptionEvent::RecoveryFailed;
  const bool ok = w.model.exceptions.fire_internal(exid, event);
  if (ok) {
    if (ex->fsm->state() == ExceptionState::Cleared ||
        !w.model.exceptions.has(exid)) {
      auto it = std::find(w.exceptions.begin(), w.exceptions.end(), exid);
      if (it != w.exceptions.end()) w.exceptions.erase(it);
    }
    w.log_action("exception_complete " + std::to_string(exid));
  }
  return ok;
}

bool act_module_event(World& w) {
  auto* id = w.pick(w.modules);
  if (!id) return false;
  static const ModuleEvent events[] = {
      ModuleEvent::StartGeneral, ModuleEvent::StartStep,
      ModuleEvent::CompleteStep, ModuleEvent::Reset,
      ModuleEvent::Abort};
  const auto ev = events[w.rng() % (sizeof(events) / sizeof(events[0]))];
  return w.model.modules.fire(*id, ev);
}

bool act_ept_event(World& w) {
  static const EptEvent events[] = {
      EptEvent::EnterNonScheduled,    EptEvent::EnterScheduledDown,
      EptEvent::EnterUnscheduledDown, EptEvent::EnterEngineering,
      EptEvent::EnterStandby,         EptEvent::EnterProductive};
  return w.model.ept.on_event(
      events[w.rng() % (sizeof(events) / sizeof(events[0]))]);
}

bool act_spool_enqueue(World& w) {
  // Spool a synthetic S6F11.  The store's policy gate accepts it only
  // if stream 6 is in the spoolable list, which we configure at startup
  // (below in the runner).
  secs2::Message m(6, 11, true);
  auto r = w.model.spool.enqueue(m);
  return r == SpoolStore::EnqueueResult::Queued;
}

bool act_spool_drain(World& w) {
  auto drained = w.model.spool.drain();
  if (drained.empty()) return false;
  w.log_action("spool_drain n=" + std::to_string(drained.size()));
  return true;
}

bool act_spool_force_toggle(World& w) {
  w.model.spool.set_force_spool(!w.model.spool.force_spool());
  return true;
}

// ---------------------------------------------------------------------------
// Invariant checks — assertions that must hold after every action.  Each
// returns a non-empty failure string on violation; empty means OK.
// ---------------------------------------------------------------------------

std::string check_invariants(World& w) {
  std::ostringstream err;

  // I-1: every tracked PJ exists in the store; conversely store size
  //      matches our active count.
  for (const auto& id : w.pjs) {
    if (!w.model.process_jobs.has(id)) {
      err << "I-1: PJ " << id << " in fuzz set but not in store; ";
      break;
    }
  }
  if (w.model.process_jobs.size() != w.pjs.size()) {
    err << "I-1: process_jobs.size()=" << w.model.process_jobs.size()
        << " != fuzz pjs.size()=" << w.pjs.size() << "; ";
  }

  // I-2: every CJ references PJs that exist in the store.
  for (const auto& id : w.cjs) {
    auto* cj = w.model.control_jobs.get(id);
    if (!cj) { err << "I-2: CJ " << id << " missing; "; continue; }
    for (const auto& pj : cj->prjobids) {
      if (!w.model.process_jobs.has(pj)) {
        err << "I-2: CJ " << id << " references missing PJ " << pj << "; ";
        break;
      }
    }
  }

  // I-3: no FSM in NoState (sentinel must not leak as live state).
  for (const auto& id : w.pjs) {
    auto* pj = w.model.process_jobs.get(id);
    if (pj && pj->fsm->state() == ProcessJobState::NoState)
      err << "I-3: PJ " << id << " in NoState; ";
  }
  for (const auto& id : w.cjs) {
    auto* cj = w.model.control_jobs.get(id);
    if (cj && cj->fsm->state() == ControlJobState::NoState)
      err << "I-3: CJ " << id << " in NoState; ";
  }

  // I-4: EPT bucket total is monotonically non-decreasing.
  uint64_t total = 0;
  for (auto s : {EptState::NonScheduledTime, EptState::ScheduledDowntime,
                  EptState::UnscheduledDowntime, EptState::Engineering,
                  EptState::Standby, EptState::Productive}) {
    total += static_cast<uint64_t>(w.model.ept.accumulated(s).count());
  }
  if (total < w.prev_ept_total_ms)
    err << "I-4: EPT total decreased from " << w.prev_ept_total_ms
        << " to " << total << "; ";
  w.prev_ept_total_ms = total;

  // I-5: defined-report VIDs all exist as SVIDs or DVIDs.
  for (const auto& r : w.model.events.all_reports()) {
    for (auto vid : r.vids) {
      if (!w.model.vid_exists(vid)) {
        err << "I-5: report " << r.id << " references unknown VID "
            << vid << "; ";
        break;
      }
    }
  }

  // I-6: substrate ids and counts.
  if (w.model.substrates.size() != w.substrates.size()) {
    err << "I-6: substrates.size()=" << w.model.substrates.size()
        << " != fuzz set " << w.substrates.size() << "; ";
  }

  // I-7: carriers.
  if (w.model.carriers.size() != w.carriers.size()) {
    err << "I-7: carriers.size()=" << w.model.carriers.size()
        << " != fuzz set " << w.carriers.size() << "; ";
  }

  return err.str();
}

// ---------------------------------------------------------------------------
// Persistence round-trip — write every persistable store to a fresh
// directory, replay into a brand-new model, and diff the in-memory
// state against the live fuzz model.
// ---------------------------------------------------------------------------

std::string check_persistence_roundtrip(World& w) {
  // Persistence was enabled in World's constructor.  Every mutation up
  // to this point has been journaled.  A fresh shadow model loading
  // from the same directory should see the same set of records.
  EquipmentDataModel shadow;
  shadow.process_jobs.set_table_factory(
      [] { return ProcessJobTransitionTable::default_table(); });
  shadow.control_jobs.set_table_factory(
      [] { return ControlJobTransitionTable::default_table(); });
  shadow.process_jobs.enable_persistence(w.journal_dir / "pj");
  shadow.control_jobs.enable_persistence(w.journal_dir / "cj");
  shadow.carriers.enable_persistence(w.journal_dir / "car");
  shadow.load_ports.enable_persistence(w.journal_dir / "lp");
  shadow.substrates.enable_persistence(w.journal_dir / "sub");
  shadow.exceptions.enable_persistence(w.journal_dir / "ex");
  shadow.spool.enable_persistence(w.journal_dir / "spool");

  std::ostringstream err;

  for (const auto& id : w.pjs) {
    auto* live = w.model.process_jobs.get(id);
    auto* shad = shadow.process_jobs.get(id);
    if (!shad) { err << "persist: PJ " << id << " missing on replay; "; continue; }
    if (live && live->fsm->state() != shad->fsm->state())
      err << "persist: PJ " << id << " state mismatch live="
          << process_job_state_name(live->fsm->state()) << " replay="
          << process_job_state_name(shad->fsm->state()) << "; ";
  }

  for (const auto& id : w.cjs) {
    auto* live = w.model.control_jobs.get(id);
    auto* shad = shadow.control_jobs.get(id);
    if (!shad) { err << "persist: CJ " << id << " missing on replay; "; continue; }
    if (live && live->fsm->state() != shad->fsm->state())
      err << "persist: CJ " << id << " state mismatch; ";
  }

  if (w.model.carriers.size() != shadow.carriers.size())
    err << "persist: carriers " << w.model.carriers.size()
        << " vs replay " << shadow.carriers.size() << "; ";
  if (w.model.substrates.size() != shadow.substrates.size())
    err << "persist: substrates " << w.model.substrates.size()
        << " vs replay " << shadow.substrates.size() << "; ";
  if (w.model.exceptions.size() != shadow.exceptions.size())
    err << "persist: exceptions " << w.model.exceptions.size()
        << " vs replay " << shadow.exceptions.size() << "; ";

  return err.str();
}

// ---------------------------------------------------------------------------
// Main driver.
// ---------------------------------------------------------------------------

struct ActionEntry {
  std::string name;
  std::function<bool(World&)> fn;
  int weight;
};

std::vector<ActionEntry> action_table() {
  return {
      {"pj_create",          act_pj_create,           8},
      {"pj_event",           act_pj_event,           14},
      {"pj_dequeue",         act_pj_dequeue,          2},
      {"cj_create",          act_cj_create,           4},
      {"cj_event",           act_cj_event,            8},
      {"cj_delete",          act_cj_delete,           2},
      {"carrier_create",     act_carrier_create,      4},
      {"carrier_id",         act_carrier_id_event,    6},
      {"carrier_slot",       act_carrier_slot_event,  4},
      {"substrate_create",   act_substrate_create,    5},
      {"substrate_loc",      act_substrate_location,  6},
      {"substrate_proc",     act_substrate_processing,6},
      {"alarm_set",          act_alarm_set,           4},
      {"alarm_clear",        act_alarm_clear,         4},
      {"alarm_enable",       act_alarm_enable_toggle, 2},
      {"svid_update",        act_svid_update,         8},
      {"define_report",      act_define_report,       2},
      {"link_event",         act_link_event,          2},
      {"enable_event",       act_enable_event,        2},
      {"exception_post",     act_exception_post,      2},
      {"exception_recover",  act_exception_recover,   2},
      {"exception_complete", act_exception_complete,  2},
      {"module_event",       act_module_event,        4},
      {"ept_event",           act_ept_event,          3},
      {"spool_enqueue",      act_spool_enqueue,       3},
      {"spool_drain",        act_spool_drain,         2},
      {"spool_force",        act_spool_force_toggle,  1},
  };
}

void run_fuzz(uint64_t seed, std::size_t ticks,
               bool with_persistence_roundtrip = true,
               std::size_t persistence_period = 500) {
  World w(seed);
  // Allow stream 5 + 6 in the spool so spool_enqueue actions land.
  w.model.spool.set_spoolable_streams({5, 6});

  const auto table = action_table();
  int total_weight = 0;
  for (const auto& a : table) total_weight += a.weight;

  for (std::size_t i = 0; i < ticks; ++i) {
    w.tick = i;
    // Weighted random action pick.
    int roll = static_cast<int>(w.rng() % static_cast<uint64_t>(total_weight));
    for (const auto& a : table) {
      if (roll < a.weight) { a.fn(w); break; }
      roll -= a.weight;
    }
    if ((i & 0x3F) == 0) {
      const auto v = check_invariants(w);
      if (!v.empty()) {
        FAIL("invariants violated at tick=" << i
              << " seed=" << seed << ": " << v << w.trace());
      }
    }
    if (with_persistence_roundtrip && i > 0 && (i % persistence_period) == 0) {
      const auto v = check_persistence_roundtrip(w);
      if (!v.empty()) {
        FAIL("persistence round-trip diverged at tick=" << i
              << " seed=" << seed << ": " << v << w.trace());
      }
    }
  }
  // Final pass.
  const auto v = check_invariants(w);
  if (!v.empty())
    FAIL("final invariants violated seed=" << seed << ": " << v << w.trace());
}

}  // namespace

TEST_CASE("Robustness fuzz: 2000 ticks × seed=0x1") {
  run_fuzz(0x1, 2000);
}

TEST_CASE("Robustness fuzz: 2000 ticks × seed=0xdeadbeef") {
  run_fuzz(0xdeadbeef, 2000);
}

TEST_CASE("Robustness fuzz: 2000 ticks × seed=0xfeedface") {
  run_fuzz(0xfeedface, 2000);
}

TEST_CASE("Robustness fuzz: 2000 ticks × seed=0xc0ffee") {
  run_fuzz(0xc0ffee, 2000);
}

TEST_CASE("Robustness fuzz: soak (gated by SECSGEM_ROBUSTNESS_SOAK=1)") {
  const char* soak = std::getenv("SECSGEM_ROBUSTNESS_SOAK");
  if (!soak || std::string(soak) != "1") {
    MESSAGE("skipping soak: set SECSGEM_ROBUSTNESS_SOAK=1 to run "
            "100k-tick robustness pass");
    return;
  }
  run_fuzz(0xa55a55a5, 100000, /*with_persistence_roundtrip=*/true,
            /*persistence_period=*/2500);
}