secs-gem/tests/test_persistence_upgrade.cpp

// Persistence schema-upgrade tests.
//
// README §6 claims: "persistence records are versioned (v1, v2) and
// forward-compatible.  Older versions still load; newer versions ignore
// unknown trailers."  This test locks down the half of that contract
// the codebase actually implements today — `ProcessJobStore` and
// `SubstrateStore` accept v1 records on current (v2) code.
//
// Craft a v1 record byte-for-byte (so the test catches accidental
// format drift), drop it into the journal directory, enable
// persistence, and assert the live store reflects the persisted state
// with the v2 trailer fields defaulted.  Then mutate, and confirm the
// record gets rewritten with the current kVersion stamp.
//
// Stores still on strict version equality (`!= kVersion`) — currently
// ControlJobStore, CarrierStore, ExceptionStore, SpoolStore — are
// exercised with a pinned v1 record + the assertion that a fabricated
// vN (N > 1) record is rejected.  When those stores grow a v2, the
// rejection assertion will need to be flipped to acceptance.

#include <doctest/doctest.h>

#include <cstdint>
#include <filesystem>
#include <fstream>
#include <random>
#include <string>
#include <vector>

#include "secsgem/gem/store/carriers.hpp"
#include "secsgem/gem/store/control_jobs.hpp"
#include "secsgem/gem/store/exceptions.hpp"
#include "secsgem/gem/store/process_jobs.hpp"
#include "secsgem/gem/store/substrates.hpp"

using namespace secsgem::gem;
namespace fs = std::filesystem;

namespace {

fs::path scratch_dir(const char* tag) {
  std::random_device rd;
  auto p = fs::temp_directory_path() /
           (std::string("secsgem-upgrade-") + tag + "-" +
            std::to_string(rd()));
  fs::remove_all(p);
  fs::create_directories(p);
  return p;
}

// Big-endian writers matching the on-disk format used by every store.
void be16(std::vector<uint8_t>& buf, uint16_t v) {
  buf.push_back(static_cast<uint8_t>(v >> 8));
  buf.push_back(static_cast<uint8_t>(v & 0xFF));
}
void put_str(std::vector<uint8_t>& buf, const std::string& s) {
  be16(buf, static_cast<uint16_t>(s.size()));
  buf.insert(buf.end(), s.begin(), s.end());
}

void write_file(const fs::path& path, const std::vector<uint8_t>& bytes) {
  std::ofstream out(path, std::ios::binary | std::ios::trunc);
  out.write(reinterpret_cast<const char*>(bytes.data()),
            static_cast<std::streamsize>(bytes.size()));
}

std::vector<uint8_t> read_file(const fs::path& path) {
  std::ifstream in(path, std::ios::binary);
  return {std::istreambuf_iterator<char>(in),
          std::istreambuf_iterator<char>()};
}

}  // namespace

TEST_CASE("Persistence upgrade: PJ v1 record loads on v2 code, no trailers") {
  auto dir = scratch_dir("pj-v1");

  // Hand-built v1 PJ record (no rcpvars / prprocessparams trailers).
  // Layout: magic=0xC7, ver=1, state, alert, mf, prrecipemethod,
  //         u16 prjobid_len + bytes, u16 ppid_len + bytes,
  //         u16 material_count + (u16 len + bytes)*.
  std::vector<uint8_t> rec;
  rec.push_back(0xC7);                                   // magic
  rec.push_back(0x01);                                   // version
  rec.push_back(static_cast<uint8_t>(ProcessJobState::Queued));
  rec.push_back(0x01);                                   // alert enabled
  rec.push_back(static_cast<uint8_t>(MaterialFlag::Substrate));
  rec.push_back(static_cast<uint8_t>(ProcessRecipeMethod::RecipeOnly));
  put_str(rec, "PJ-LEGACY");
  put_str(rec, "RECIPE-V1");
  be16(rec, 2);
  put_str(rec, "WFR-1");
  put_str(rec, "WFR-2");
  write_file(dir / "0000000001.pj", rec);

  {
    ProcessJobStore s;
    s.enable_persistence(dir);
    REQUIRE(s.has("PJ-LEGACY"));
    auto* pj = s.get("PJ-LEGACY");
    REQUIRE(pj);
    CHECK(pj->ppid == "RECIPE-V1");
    CHECK(pj->fsm->state() == ProcessJobState::Queued);
    CHECK(pj->alert_enabled);
    CHECK(pj->mf == MaterialFlag::Substrate);
    CHECK(pj->prrecipemethod == ProcessRecipeMethod::RecipeOnly);
    CHECK(pj->mtrloutspec == std::vector<std::string>{"WFR-1", "WFR-2"});
    // v1 records carry no extras; the v2 fields come back empty.
    CHECK(pj->rcpvars.empty());
    CHECK(pj->prprocessparams.empty());
  }

  // Any mutation rewrites the record — the new file should carry the
  // current kVersion stamp.  The cheapest mutation is set_alert.
  {
    ProcessJobStore s;
    s.enable_persistence(dir);
    REQUIRE(s.set_alert("PJ-LEGACY", false));
  }
  auto on_disk = read_file(dir / "0000000001.pj");
  REQUIRE(on_disk.size() >= 2);
  CHECK(on_disk[0] == 0xC7);
  CHECK(on_disk[1] == 0x02);  // upgraded to current version stamp

  fs::remove_all(dir);
}

TEST_CASE("Persistence upgrade: PJ rejects an unknown future version") {
  auto dir = scratch_dir("pj-future");
  // Same byte layout as v1, but stamped as v9 — represents a record
  // written by a hypothetical future binary the current code can't
  // decode.  The current strict allow-list (1 or 2) should reject.
  std::vector<uint8_t> rec;
  rec.push_back(0xC7);
  rec.push_back(0x09);
  rec.push_back(static_cast<uint8_t>(ProcessJobState::Queued));
  rec.push_back(0);
  rec.push_back(static_cast<uint8_t>(MaterialFlag::Substrate));
  rec.push_back(static_cast<uint8_t>(ProcessRecipeMethod::RecipeOnly));
  put_str(rec, "PJ-FUTURE");
  put_str(rec, "RECIPE");
  be16(rec, 0);
  write_file(dir / "0000000001.pj", rec);

  ProcessJobStore s;
  s.enable_persistence(dir);
  CHECK_FALSE(s.has("PJ-FUTURE"));
  // Rejected records are deleted (corrupt-drop behaviour), matching
  // the existing partial-write recovery test.
  CHECK_FALSE(fs::exists(dir / "0000000001.pj"));
  fs::remove_all(dir);
}

TEST_CASE("Persistence upgrade: Substrate v1 record loads, history empty") {
  auto dir = scratch_dir("sub-v1");

  std::vector<uint8_t> rec;
  rec.push_back(0xC6);                                  // magic
  rec.push_back(0x01);                                  // version
  rec.push_back(static_cast<uint8_t>(SubstrateState::AtSource));
  rec.push_back(static_cast<uint8_t>(SubstrateProcessingState::NeedsProcessing));
  rec.push_back(static_cast<uint8_t>(SubstrateIDStatus::NotConfirmed));
  rec.push_back(/*slot=*/3);
  put_str(rec, "W-LEGACY-001");
  put_str(rec, "CAR-A1B2");
  put_str(rec, "LoadPort1");
  write_file(dir / "0000000001.sub", rec);

  SubstrateStore s;
  s.enable_persistence(dir);
  REQUIRE(s.has("W-LEGACY-001"));
  auto* sub = s.get("W-LEGACY-001");
  REQUIRE(sub);
  CHECK(sub->carrierid == "CAR-A1B2");
  CHECK(sub->slot == 3);
  CHECK(sub->location == "LoadPort1");
  CHECK(sub->fsm->location_state() == SubstrateState::AtSource);
  CHECK(sub->fsm->processing_state() ==
        SubstrateProcessingState::NeedsProcessing);
  // History is the v2 trailer — v1 records replay with an empty vector.
  const auto* hist = s.history("W-LEGACY-001");
  REQUIRE(hist);
  CHECK(hist->empty());
  fs::remove_all(dir);
}

// --- Multi-version-aware stores: all stores now accept v in [1, kVersion]
// --- following the PJ/Substrate pattern.  Future kVersion bumps need
// --- to append fields behind `if (version >= N)` gates; the loaders
// --- already accept the prior versions.  These tests pin the contract.

TEST_CASE("Persistence upgrade: ControlJobStore rejects a future version") {
  auto dir = scratch_dir("cj-future");
  std::vector<uint8_t> rec;
  rec.push_back(0xC8);
  rec.push_back(0x09);  // not the current kVersion
  rec.push_back(static_cast<uint8_t>(ControlJobState::Queued));
  put_str(rec, "CJ-FUTURE");
  be16(rec, 0);
  write_file(dir / "0000000001.cj", rec);

  ControlJobStore s;
  s.enable_persistence(dir);
  CHECK_FALSE(s.has("CJ-FUTURE"));
  fs::remove_all(dir);
}

TEST_CASE("Persistence upgrade: CarrierStore rejects a future version") {
  auto dir = scratch_dir("car-future");
  // Carrier records start with magic = 0xC4 — confirm the version
  // gate kicks in even when magic matches.
  std::vector<uint8_t> rec;
  rec.push_back(0xC4);
  rec.push_back(0x09);
  // Pad with zeros so the loader's reads don't underflow before the
  // version reject point.
  for (int i = 0; i < 64; ++i) rec.push_back(0);
  write_file(dir / "0000000001.car", rec);

  CarrierStore s;
  s.enable_persistence(dir);
  CHECK(s.size() == 0);
  fs::remove_all(dir);
}

TEST_CASE("Persistence upgrade: ControlJobStore loads v1 record") {
  // Positive case: v1 record matches the current schema exactly, but
  // proves the loader's range check (1 <= version <= kVersion) lets
  // the actual current version through.  When kVersion bumps to 2,
  // *this* test continues to assert that v1 still replays.
  auto dir = scratch_dir("cj-v1");
  std::vector<uint8_t> rec;
  rec.push_back(0xC8);
  rec.push_back(0x01);
  rec.push_back(static_cast<uint8_t>(ControlJobState::Queued));
  put_str(rec, "CJ-V1");
  be16(rec, 1);
  put_str(rec, "PJ-V1");
  write_file(dir / "0000000001.cj", rec);

  ControlJobStore s;
  s.enable_persistence(dir);
  REQUIRE(s.has("CJ-V1"));
  auto* cj = s.get("CJ-V1");
  REQUIRE(cj);
  CHECK(cj->prjobids == std::vector<std::string>{"PJ-V1"});
  CHECK(cj->fsm->state() == ControlJobState::Queued);
  fs::remove_all(dir);
}

TEST_CASE("Persistence upgrade: ExceptionStore rejects a future version") {
  auto dir = scratch_dir("ex-future");
  std::vector<uint8_t> rec;
  rec.push_back(0xC9);
  rec.push_back(0x09);
  for (int i = 0; i < 64; ++i) rec.push_back(0);
  write_file(dir / "0000000001.ex", rec);

  ExceptionStore s;
  s.enable_persistence(dir);
  CHECK(s.size() == 0);
  fs::remove_all(dir);
}