robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(60A.7): add reusable decision-options fixture loading in test harness

Browse Source 
- Load decisions-options fixture from committed JSON (tests/fixtures/
  pre-anchored-decision-options.json) instead of inline duplicate
- Add runPreAnchoredSimulationWithFixture() helper for decision-options
  mode tests
- Generalize anchor validation from savings-realism-specific to generic
  unresolved unknown check in reproduce-multi-turn-investigation.mjs
- Add experiment documentation (experiment-60a7.md) and handoff note
- All 63 harness tests pass; no production reasoning code changed
This commit is contained in:
robbond
2026-08-13 06:24:12 +01:00
parent 2016a024c5
commit 4c25faaa01
5 changed files with 634 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/reproduce-multi-turn-investigation.mjs
+34 -16
View File
@@ -2,11 +2,24 @@ import fs from "fs";
import { fileURLToPath } from "url";
import path from "path";
const FIXTURE_PATH = path.resolve(
// Default fixture — used when FIXTURE_PATH is not set (existing behaviour).
const DEFAULT_FIXTURE_PATH = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"../tests/fixtures/pre-anchored-update-savings-realism.json",
);
/**
* Resolves the fixture path for updateOnly mode.
* FIXTURE_PATH (env) overrides the default if set and non-empty.
*/
function resolveFixturePath() {
const envPath = process.env.FIXTURE_PATH;
if (envPath && String(envPath).trim() !== "") {
return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", envPath);
}
return DEFAULT_FIXTURE_PATH;
}
const BASE_URL =
process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";
@@ -201,13 +214,15 @@ async function runUpdateOnlyMode() {
return; // zero live calls made
}
const fixturePath = resolveFixturePath();
// Load committed fixture — single source of truth.
let fixtureData;
try {
const raw = fs.readFileSync(FIXTURE_PATH, "utf-8");
const raw = fs.readFileSync(fixturePath, "utf-8");
fixtureData = JSON.parse(raw);
} catch (err) {
console.error(`ERROR: Cannot load pre-anchored fixture from ${FIXTURE_PATH}`);
console.error(`ERROR: Cannot load pre-anchored fixture from ${fixturePath}`);
process.exitCode = 1;
return;
}
@@ -217,33 +232,36 @@ async function runUpdateOnlyMode() {
// Verify fixture integrity before proceeding.
const nodes = initialGraph.nodes;
const edges = initialGraph.edges;
const savingsNodes = nodes.filter(
(n) => n.kind === "unknown" && n.status === "unknown" && n.label.includes("savings"),
// Generic anchor check: any unresolved unknown node (supports both savings-realism and decision-options fixtures).
const unresolvedNodes = nodes.filter(
(n) => n.kind === "unknown" && n.status === "unknown",
);
if (savingsNodes.length !== 1) {
console.log(`ERROR: pre-anchored fixture does not contain exactly one savings-realism anchor (found ${savingsNodes.length}).`);
if (unresolvedNodes.length < 1) {
console.log(`ERROR: pre-anchored fixture must contain at least one unresolved unknown node (found ${unresolvedNodes.length}).`);
process.exitCode = 1;
return;
}
// Use the first unresolved unknown as the anchor for previousQuestion derivation.
const anchorNode = unresolvedNodes[0];
// Pre-call compact fixture snapshot.
const savingsNode = savingsNodes[0];
console.log(`\n=== UPDATE-ONLY MODE ===`);
console.log(`fixture node count: ${nodes.length}`);
console.log(`fixture edge count: ${edges.length}`);
console.log(`savings-realism node:`);
console.log(` id: ${savingsNode.id}`);
console.log(` label: ${savingsNode.label}`);
console.log(` kind: ${savingsNode.kind}`);
console.log(` status: ${savingsNode.status}`);
console.log(`anchor node:`);
console.log(` id: ${anchorNode.id}`);
console.log(` label: ${anchorNode.label}`);
console.log(` kind: ${anchorNode.kind}`);
console.log(` status: ${anchorNode.status}`);
// Deep-copy so mutation doesn't corrupt the original fixture.
const situationGraph = JSON.parse(JSON.stringify(initialGraph));
// Derive previousQuestion from the committed savings-realism anchor so that
// update-only mode can reach the production Update path without requiring Start.
let selectedQuestion = fixtureData.unresolved_question ?? savingsNode.label;
// Derive previousQuestion from the fixture's unresolved_question or anchor node label.
let selectedQuestion = fixtureData.unresolved_question ?? anchorNode.label;
// ── Exactly one Update through production route ────────
const updateNum = 1;