robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(experiments): expose reconstruction open-question evidence

Browse Source
This commit is contained in:
robbond
2026-10-09 08:12:06 +01:00
parent 6bc8be6edb
commit fd5241c6af
2 changed files with 255 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-case-experiment-helper.cjs
+88 -3
View File
@@ -67,6 +67,12 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
let reconstructionModelName = options.reconstructionModelName;
let reconstructionOnly = options.reconstructionOnly;
const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";
const productionRequest = {
scenario: scenarioInput?.scenario,
...(scenarioInput?.promptVersion !== undefined
? { promptVersion: scenarioInput.promptVersion }
: {}),
};
if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
if (!process.env.OPENAI_API_KEY) {
@@ -95,13 +101,51 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
}
return {
success: true,
situationGraph: { nodes: [], edges: [], activeUnknownNodeId: null, resolvedNodeIds: [], currentSummary: "test" },
reconstruction: {
importantUnknowns: [
{
id: "unknown-1",
description: "Canonical uncertainty from the deterministic test double",
confidence: "medium",
question: "What would resolve the deterministic uncertainty?",
},
],
},
situationGraph: {
nodes: [
{
id: "graph-unknown-1",
label: "Canonical uncertainty",
description: "Canonical uncertainty from the deterministic test double",
openQuestion: "What would resolve the deterministic uncertainty?",
status: "unknown",
kind: "unknown",
},
{
id: "graph-state-1",
label: "Known state",
description: "A non-unknown node",
status: "supported",
kind: "state",
},
],
edges: [],
activeUnknownNodeId: null,
resolvedNodeIds: [],
currentSummary: "test",
},
assessment: { phase: "initial", progress: 0 },
selectedQuestion: null,
summary: null,
diagnostics: {
validationStatus: "mock",
modelName: "inline-test-double",
promptVersion: productionRequest.promptVersion ?? "v0.5",
presentationValidationStatus: "valid",
presentationValidationIssues: [],
compatibilityApplied: false,
compatibilityChanges: [],
compatibilityWarnings: [],
reconstructionOnly: dependencies.reconstructionOnly === true,
graphReferenceValidation: { valid: true, errors: [] },
},
@@ -125,7 +169,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
let result;
const endToEndStartedAt = Date.now();
try {
result = await startCase(scenarioInput, {
result = await startCase(productionRequest, {
reconstructionProvider,
reconstructionModelName,
reconstructionOnly,
@@ -147,7 +191,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
}
}
return {
const output = {
success: result.success,
summary: result.summary ?? null,
reconstruction: result.reconstruction ?? null,
@@ -162,6 +206,47 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
error: result.success ? null : (result.error ?? "unknown"),
statusCode: result.statusCode ?? (result.success ? 200 : 500),
};
if (reconstructionOnly) {
if (scenarioInput?.experimentId !== undefined) {
output.experimentId = scenarioInput.experimentId;
}
if (scenarioInput?.scenarioId !== undefined) {
output.scenarioId = scenarioInput.scenarioId;
}
const configuredProvider =
process.env.START_CASE_EXPERIMENT_PROVIDER === "openai" ? "openai" : "ollama";
const configuredModel =
result.diagnostics?.modelName ??
reconstructionModelName ??
(configuredProvider === "ollama" ? process.env.OLLAMA_MODEL : undefined);
const effectivePromptVersion = result.diagnostics?.promptVersion;
output.execution = {
provider: configuredProvider,
...(configuredModel !== undefined ? { model: configuredModel } : {}),
...(effectivePromptVersion !== undefined
? { promptVersion: effectivePromptVersion }
: {}),
};
if (Array.isArray(result.situationGraph?.nodes)) {
output.graphUnknownNodes = result.situationGraph.nodes
.filter((node) => node.kind === "unknown")
.map((node) => ({
id: node.id,
label: node.label,
description: node.description,
...(node.openQuestion !== undefined
? { openQuestion: node.openQuestion }
: {}),
...(node.status !== undefined ? { status: node.status } : {}),
}));
}
}
return output;
}
if (require.main === module) {
tests/scripts/start-case-experiment-helper.test.js
+167 -1
View File
@@ -11,7 +11,7 @@
import { describe, expect, it, beforeAll, afterAll, vi } from "vitest";
import { execFile } from "child_process";
import { writeFile, unlink } from "fs/promises";
import { readFile, writeFile, unlink } from "fs/promises";
import { join, dirname } from "path";
import { fileURLToPath } from "url";
import { createRequire } from "module";
@@ -89,16 +89,29 @@ function parseOutput(result) {
describe("start-case-experiment-helper.cjs apparatus", () => {
let tmpFile = "";
let metadataFile = "";
beforeAll(async () => {
// Create a temporary scenario fixture for file-input tests
tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json");
await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8");
metadataFile = join(rootDir, "tests", "fixtures", "_helper_semantic_scenario_temp.json");
await writeFile(
metadataFile,
JSON.stringify({
experimentId: "semantic-open-question-001",
scenarioId: "manufacturing-expansion-001",
scenario: "Scenario with semantic inspection metadata",
promptVersion: "v0.5",
}),
"utf-8",
);
});
afterAll(async () => {
// Clean up temp fixture
try { await unlink(tmpFile); } catch {}
try { await unlink(metadataFile); } catch {}
});
// ── A — Positional scenario input ───────────────────────────────
@@ -157,6 +170,141 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
expect(result.exitCode).toBe(0);
expect(parseOutput(result).diagnostics.reconstructionOnly).toBe(true);
});
it("preserves experiment metadata and exposes semantic evidence", async () => {
const result = await runHelper(["--reconstruction-only", "--file", metadataFile]);
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed.experimentId).toBe("semantic-open-question-001");
expect(parsed.scenarioId).toBe("manufacturing-expansion-001");
expect(parsed.execution).toEqual({
provider: "ollama",
model: "inline-test-double",
promptVersion: "v0.5",
});
expect(parsed.reconstruction.importantUnknowns).toEqual([
{
id: "unknown-1",
description: "Canonical uncertainty from the deterministic test double",
confidence: "medium",
question: "What would resolve the deterministic uncertainty?",
},
]);
expect(parsed.graphUnknownNodes).toEqual([
{
id: "graph-unknown-1",
label: "Canonical uncertainty",
description: "Canonical uncertainty from the deterministic test double",
openQuestion: "What would resolve the deterministic uncertainty?",
status: "unknown",
},
]);
expect(parsed.diagnostics).toMatchObject({
validationStatus: "mock",
presentationValidationStatus: "valid",
presentationValidationIssues: [],
graphReferenceValidation: { valid: true, errors: [] },
compatibilityApplied: false,
compatibilityChanges: [],
compatibilityWarnings: [],
});
});
it("passes only production-supported request fields and calls startCase once", async () => {
const { runStartCaseExperiment } = require(helperPath);
const importantUnknowns = [
{
id: "unknown-a",
description: "Canonical uncertainty A",
confidence: "high",
question: "What resolves uncertainty A?",
},
];
const startCase = vi.fn(async () => ({
success: true,
reconstruction: { importantUnknowns },
situationGraph: {
nodes: [
{
id: "unknown-node-a",
label: "Canonical uncertainty A",
description: "Canonical uncertainty A",
openQuestion: "What resolves uncertainty A?",
status: "unknown",
kind: "unknown",
},
{
id: "observation-node-a",
label: "Known observation",
description: "Known observation",
status: "supported",
kind: "observation",
},
],
edges: [],
},
selectedQuestion: null,
summary: "test",
diagnostics: {
modelName: "configured-model",
promptVersion: "v0.5",
validationStatus: "valid",
presentationValidationStatus: "partial",
presentationValidationIssues: [{ category: "test-diagnostic" }],
graphReferenceValidation: { valid: true, errors: [] },
compatibilityApplied: true,
compatibilityChanges: ["normalised"],
compatibilityWarnings: ["warning"],
},
}));
const result = await runStartCaseExperiment(
{
experimentId: "experiment-a",
scenarioId: "scenario-a",
scenario: "Scenario A",
promptVersion: "v0.5",
ignoredMetadata: "must not cross the production boundary",
},
null,
{ startCase, reconstructionOnly: true },
);
expect(startCase).toHaveBeenCalledTimes(1);
expect(startCase).toHaveBeenCalledWith(
{ scenario: "Scenario A", promptVersion: "v0.5" },
expect.objectContaining({ reconstructionOnly: true }),
);
expect(result.experimentId).toBe("experiment-a");
expect(result.scenarioId).toBe("scenario-a");
expect(result.reconstruction.importantUnknowns).toBe(importantUnknowns);
expect(result.graphUnknownNodes).toEqual([
{
id: "unknown-node-a",
label: "Canonical uncertainty A",
description: "Canonical uncertainty A",
openQuestion: "What resolves uncertainty A?",
status: "unknown",
},
]);
expect(result.execution).toEqual({
provider: "ollama",
model: "configured-model",
promptVersion: "v0.5",
});
expect(result.diagnostics).toEqual({
modelName: "configured-model",
promptVersion: "v0.5",
validationStatus: "valid",
presentationValidationStatus: "partial",
presentationValidationIssues: [{ category: "test-diagnostic" }],
graphReferenceValidation: { valid: true, errors: [] },
compatibilityApplied: true,
compatibilityChanges: ["normalised"],
compatibilityWarnings: ["warning"],
});
});
});
// ── C — Environment loading ─────────────────────────────────────
@@ -295,6 +443,19 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
const parsed = parseOutput(result);
expect(parsed.error).toBeNull();
});
it("preserves the existing non-reconstruction-only output contract", async () => {
const result = await runHelper(["Compatibility scenario"]);
const parsed = parseOutput(result);
expect(parsed).not.toHaveProperty("experimentId");
expect(parsed).not.toHaveProperty("scenarioId");
expect(parsed).not.toHaveProperty("execution");
expect(parsed).not.toHaveProperty("graphUnknownNodes");
expect(parsed).toHaveProperty("situationGraphNodeCount");
expect(parsed).toHaveProperty("situationGraphEdgeCount");
expect(parsed).toHaveProperty("endToEndElapsedMs");
});
});
// ── E — Failure output (startCase throws) ───────────────────────
@@ -399,6 +560,11 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
expect(typeof parsed.error).toBe("string");
}
});
it("contains no investigation-persistence invocation", async () => {
const source = await readFile(helperPath, "utf-8");
expect(source).not.toMatch(/investigation-storage|server-investigation-persistence|saveInvestigation|\/api\/investigations/);
});
});
// ── H — Import-only mode (production seam verification) ────────