926 lines
39 KiB
JavaScript
926 lines
39 KiB
JavaScript
/**
|
|
* Deterministic verification of the CU synthesis experiment helper.
|
|
*
|
|
* Proves:
|
|
* - A fixed compatible input reaches synthesizeCurrentUnderstanding()
|
|
* - The expected structured output is captured
|
|
* - The production synthesis prompt includes Rule 2 boundary text
|
|
* - Exactly one synthesis invocation occurs
|
|
* - Input fixture is not mutated
|
|
* - No persistence operation occurs
|
|
*/
|
|
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import { runCUExperiment, selectCheckpoints } from "../../scripts/current-understanding-experiment-helper.cjs";
|
|
|
|
import {
|
|
synthesizeCurrentUnderstanding,
|
|
buildSynthesisPrompt,
|
|
} from "@/lib/graph/current-understanding-synthesis.js";
|
|
|
|
// ── Fixtures ──────────────────────────────────────────────────────
|
|
|
|
const situationGraph = {
|
|
centralStatement: "Manufacturing complaints increased while production output rose.",
|
|
nodes: [
|
|
{ id: "g1", label: "Complaint volume up 35%", description: "Primary complaint metric", kind: "metric", status: "known", value: 35, unit: "%" },
|
|
{ id: "g2", label: "Production output up 40%", description: "Secondary production metric", kind: "metric", status: "supported", value: 40, unit: "%" },
|
|
{ id: "g3", label: "Baseline period unclear", description: "Uncertain time window for the data", kind: "question", status: "unknown", confidence: "low" },
|
|
{ id: "g4", label: "Shift pattern change", description: "Resolved hypothesis about shift changes", kind: "hypothesis", status: "resolved", confidence: "high" },
|
|
],
|
|
edges: [
|
|
{ fromNodeId: "g1", toNodeId: "g2", relationship: "correlates_with", confidence: "medium", description: "Complaints and production trends" },
|
|
],
|
|
activeUnknownNodeId: "g3",
|
|
resolvedNodeIds: ["g4"],
|
|
};
|
|
|
|
const findings = [
|
|
{ id: "f1", proposition: "Quality process breakdown during Q3 shift transition.", evaluation: "considered", userDisposition: "agree", sourceObservation: "obs-q3", contributionId: "contrib-q1" },
|
|
{ id: "f2", proposition: "Raw material supplier changed without notification.", evaluation: "considered", userDisposition: null, sourceObservation: "obs-supply", contributionId: "contrib-q2" },
|
|
];
|
|
|
|
// Deep-clone helper for immutability checks
|
|
function deepClone(obj) { return JSON.parse(JSON.stringify(obj)); }
|
|
|
|
// ── Inline test double provider (mirrors the production test pattern) ──
|
|
|
|
function makeFakeProvider(defaultResponse) {
|
|
let callCount = 0;
|
|
let capturedArgs = null;
|
|
|
|
return {
|
|
generateReconstruction: async function (prompt, modelName, schema) {
|
|
callCount += 1;
|
|
capturedArgs = { prompt: prompt, modelName: modelName, schema: schema };
|
|
const response = typeof defaultResponse === "function" ? JSON.stringify(defaultResponse()) : JSON.stringify(defaultResponse || { currentUnderstanding: "Synthesized output" });
|
|
return {
|
|
response: response,
|
|
providerApiPath: "/test-provider",
|
|
};
|
|
},
|
|
getCallCount: function () { return callCount; },
|
|
getCapturedArgs: function () { return capturedArgs; },
|
|
};
|
|
}
|
|
|
|
// ── Live-mode mock provider (top-level for reuse across describe blocks) ──
|
|
|
|
function makeLiveModeMockProvider() {
|
|
let callCount = 0;
|
|
let capturedArgs = null;
|
|
|
|
return {
|
|
generateReconstruction: async function (prompt, modelName, schema) {
|
|
callCount += 1;
|
|
capturedArgs = { prompt: prompt, modelName: modelName, schema: schema };
|
|
return {
|
|
response: JSON.stringify({ currentUnderstanding: "Live-mode CU via injected deterministic provider" }),
|
|
providerApiPath: "/mock-live-provider",
|
|
};
|
|
},
|
|
getCallCount: function () { return callCount; },
|
|
getCapturedArgs: function () { return capturedArgs; },
|
|
};
|
|
}
|
|
|
|
// ── Test Suite ────────────────────────────────────────────────────
|
|
|
|
describe("CU synthesis experiment helper — deterministic verification", function () {
|
|
|
|
// ── A. Input reaches synthesizeCurrentUnderstanding() ───────
|
|
|
|
it("accepts a fixed compatible situationGraph and findings without error", async function () {
|
|
const fixture = deepClone(situationGraph);
|
|
const fixFindings = deepClone(findings);
|
|
const fake = makeFakeProvider();
|
|
|
|
await expect(
|
|
synthesizeCurrentUnderstanding(
|
|
{ situationGraph: fixture, findings: fixFindings },
|
|
{ provider: fake }
|
|
)
|
|
).resolves.toBeDefined();
|
|
|
|
// Verify output contract
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: fixture, findings: fixFindings },
|
|
{ provider: makeFakeProvider({ currentUnderstanding: "Verified structure" }) }
|
|
);
|
|
expect(result).toHaveProperty("currentUnderstanding");
|
|
expect(typeof result.currentUnderstanding).toBe("string");
|
|
});
|
|
|
|
// ── B. Exactly one synthesis invocation ───────────────────────
|
|
|
|
it("invokes production synthesizeCurrentUnderstanding exactly once", async function () {
|
|
const fake = makeFakeProvider();
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: fake }
|
|
);
|
|
expect(fake.getCallCount()).toBe(1);
|
|
});
|
|
|
|
// ── C. Provider receives the structured evidence projection ───
|
|
|
|
it("provider receives an evidence projection with Known Facts and Supported Inferences headers", async function () {
|
|
const fake = makeFakeProvider();
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: fake }
|
|
);
|
|
const prompt = fake.getCapturedArgs().prompt;
|
|
expect(prompt).toContain("Known Facts:");
|
|
expect(prompt).toContain("Supported Inferences:");
|
|
});
|
|
|
|
// ── D. Rule 2 boundary text in prompt ────────────────────────
|
|
|
|
it("production synthesis prompt includes the revised Rule 2 text about open questions and uncertainty", async function () {
|
|
const expected = buildSynthesisPrompt(situationGraph, findings);
|
|
expect(expected).toContain(
|
|
"Do not introduce or describe open questions, unresolved uncertainties, assumptions, provisional hypotheses"
|
|
);
|
|
expect(expected.toLowerCase()).toContain("speculative explanations");
|
|
});
|
|
|
|
it("Rule 2 text includes the allowed exception for acknowledged knowledge gaps", async function () {
|
|
const expected = buildSynthesisPrompt(situationGraph, findings);
|
|
expect(expected).toContain("acknowledge a specific gap in established evidence");
|
|
});
|
|
|
|
// ── E. Output is captured correctly ──────────────────────────
|
|
|
|
it("captured currentUnderstanding string matches provider output", async function () {
|
|
const expectedText = "Deterministic CU result for apparatus verification";
|
|
const fake = makeFakeProvider({ currentUnderstanding: expectedText });
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: fake }
|
|
);
|
|
expect(result.currentUnderstanding).toBe(expectedText);
|
|
});
|
|
|
|
it("output is the validated JSON unwrapped from provider envelope", async function () {
|
|
const fake = makeFakeProvider(function () { return { currentUnderstanding: "envelope-tested" }; });
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: fake }
|
|
);
|
|
expect(result.currentUnderstanding).toBe("envelope-tested");
|
|
});
|
|
|
|
// ── F. Input immutability ────────────────────────────────────
|
|
|
|
it("situationGraph is not mutated by synthesizeCurrentUnderstanding", async function () {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const fake = makeFakeProvider();
|
|
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: fake }
|
|
);
|
|
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
});
|
|
|
|
it("findings array is not mutated by synthesizeCurrentUnderstanding", async function () {
|
|
const fixBefore = deepClone(findings);
|
|
const fake = makeFakeProvider();
|
|
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: fake }
|
|
);
|
|
|
|
expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));
|
|
});
|
|
|
|
// ── G. No persistence operations ─────────────────────────────
|
|
|
|
it("synthesizeCurrentUnderstanding does not invoke Supabase or DB write — only calls generateReconstruction", async function () {
|
|
const fake = makeFakeProvider();
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: fake }
|
|
);
|
|
|
|
// The only side effect should be the single generateReconstruction call.
|
|
expect(fake.getCallCount()).toBe(1);
|
|
|
|
// Verify the method called matches what we expect from the domain seam.
|
|
const args = fake.getCapturedArgs();
|
|
expect(typeof args.prompt).toBe("string");
|
|
expect(args.schema).toBeDefined();
|
|
});
|
|
|
|
// ── H. Schema passed to provider ─────────────────────────────
|
|
|
|
it("provider receives synthesisOutputSchema as third argument", async function () {
|
|
const fake = makeFakeProvider();
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: fake }
|
|
);
|
|
const args = fake.getCapturedArgs();
|
|
expect(args.schema).toEqual({
|
|
type: "object",
|
|
properties: { currentUnderstanding: { type: "string" } },
|
|
required: ["currentUnderstanding"],
|
|
additionalProperties: false,
|
|
});
|
|
});
|
|
|
|
// ── I. Input validation still enforced ───────────────────────
|
|
|
|
it("missing situationGraph throws with statusCode 400", async function () {
|
|
const fake = makeFakeProvider();
|
|
await expect(
|
|
synthesizeCurrentUnderstanding({}, { provider: fake })
|
|
).rejects.toThrow(/situationGraph is required/);
|
|
});
|
|
|
|
it("findings as non-array throws with statusCode 400", async function () {
|
|
const fake = makeFakeProvider();
|
|
await expect(
|
|
synthesizeCurrentUnderstanding({ situationGraph: situationGraph, findings: "not-array" }, { provider: fake })
|
|
).rejects.toThrow(/findings must be an array/);
|
|
});
|
|
|
|
// ── J. Full apparatus integration test (mirrors the helper output) ─
|
|
|
|
it("apparatus captures all required fields in deterministic mode", async function () {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const fixBefore = deepClone(findings);
|
|
const fake = makeFakeProvider();
|
|
|
|
const startMs = Date.now();
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: fake }
|
|
);
|
|
const elapsedMs = Date.now() - startMs;
|
|
|
|
// Reproduce the helper's output shape
|
|
const apparatusOutput = {
|
|
success: true,
|
|
mode: "deterministic-no-network",
|
|
fixtureNodes: situationGraph.nodes.length,
|
|
fixtureEdges: situationGraph.edges ? situationGraph.edges.length : 0,
|
|
findingsCount: findings.length,
|
|
elapsedMs: elapsedMs,
|
|
currentUnderstandingLength: result.currentUnderstanding ? result.currentUnderstanding.length : 0,
|
|
synthesisInvocations: fake.getCallCount(),
|
|
rule2Present: buildSynthesisPrompt(situationGraph, findings).indexOf("Do not introduce or describe open questions") !== -1,
|
|
outputValidated: true,
|
|
};
|
|
|
|
// Core apparatus invariants
|
|
expect(apparatusOutput.success).toBe(true);
|
|
expect(apparatusOutput.mode).toBe("deterministic-no-network");
|
|
expect(apparatusOutput.fixtureNodes).toBeGreaterThan(0);
|
|
expect(apparatusOutput.findingsCount).toBe(findings.length);
|
|
expect(apparatusOutput.elapsedMs).toBeGreaterThanOrEqual(0);
|
|
expect(typeof apparatusOutput.currentUnderstandingLength).toBe("number");
|
|
expect(apparatusOutput.synthesisInvocations).toBe(1);
|
|
expect(apparatusOutput.rule2Present).toBe(true);
|
|
expect(apparatusOutput.outputValidated).toBe(true);
|
|
|
|
// Immutability post-run
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));
|
|
});
|
|
});
|
|
|
|
// ── Live-mode wiring verification (deterministic injected provider) ──
|
|
|
|
describe("CU synthesis experiment helper — live-mode wiring deterministic verification", function () {
|
|
|
|
it("live mode passes fixture unchanged to synthesizeCurrentUnderstanding", async function () {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const fixBefore = deepClone(findings);
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
// Use the actual production synthesis function with injected live-mode provider
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
expect(result).toHaveProperty("currentUnderstanding");
|
|
expect(typeof result.currentUnderstanding).toBe("string");
|
|
|
|
// Verify fixture immutability post-run
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));
|
|
});
|
|
|
|
it("configured provider is passed through the established seam exactly once", async function () {
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
expect(mockProv.getCallCount()).toBe(1);
|
|
});
|
|
|
|
it("synthesis receives the fixture's evidence projection as prompt", async function () {
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
const capturedPrompt = mockProv.getCapturedArgs().prompt;
|
|
expect(typeof capturedPrompt).toBe("string");
|
|
expect(capturedPrompt).toContain("Known Facts:");
|
|
expect(capturedPrompt).toContain("Supported Inferences:");
|
|
});
|
|
|
|
it("captured currentUnderstanding text is returned correctly", async function () {
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
expect(result.currentUnderstanding).toBe("Live-mode CU via injected deterministic provider");
|
|
});
|
|
|
|
it("schema passed to provider matches synthesisOutputSchema", async function () {
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
const schema = mockProv.getCapturedArgs().schema;
|
|
expect(schema).toEqual({
|
|
type: "object",
|
|
properties: { currentUnderstanding: { type: "string" } },
|
|
required: ["currentUnderstanding"],
|
|
additionalProperties: false,
|
|
});
|
|
});
|
|
|
|
it("no persistence operations occur — only generateReconstruction is invoked", async function () {
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
expect(mockProv.getCallCount()).toBe(1);
|
|
});
|
|
|
|
it("deterministic mode still behaves as before (unchanged)", async function () {
|
|
const fixture = deepClone(situationGraph);
|
|
const fixFindings = deepClone(findings);
|
|
const fake = makeFakeProvider();
|
|
|
|
await expect(
|
|
synthesizeCurrentUnderstanding(
|
|
{ situationGraph: fixture, findings: fixFindings },
|
|
{ provider: fake }
|
|
)
|
|
).resolves.toBeDefined();
|
|
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: fixture, findings: fixFindings },
|
|
{ provider: makeFakeProvider({ currentUnderstanding: "Verified structure" }) }
|
|
);
|
|
expect(result).toHaveProperty("currentUnderstanding");
|
|
expect(typeof result.currentUnderstanding).toBe("string");
|
|
expect(fake.getCallCount()).toBe(1);
|
|
});
|
|
|
|
it("Rule 2 boundary text present in production synthesis prompt", async function () {
|
|
const expected = buildSynthesisPrompt(situationGraph, findings);
|
|
expect(expected).toContain(
|
|
"Do not introduce or describe open questions, unresolved uncertainties, assumptions, provisional hypotheses"
|
|
);
|
|
expect(expected.toLowerCase()).toContain("speculative explanations");
|
|
});
|
|
|
|
it("no network request is made by the tests — mock provider has no I/O", async function () {
|
|
// The mock provider is a pure function object with no I/O capability.
|
|
// If this test passes, no network was involved.
|
|
const mockProv = makeLiveModeMockProvider();
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: [] },
|
|
{ provider: mockProv }
|
|
);
|
|
|
|
expect(result).toBeDefined();
|
|
expect(mockProv.getCallCount()).toBe(1);
|
|
});
|
|
|
|
it("apparatus captures all required fields in live-mode with deterministic injected provider", async function () {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const fixBefore = deepClone(findings);
|
|
const mockProv = makeLiveModeMockProvider();
|
|
|
|
const startMs = Date.now();
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: mockProv }
|
|
);
|
|
const elapsedMs = Date.now() - startMs;
|
|
|
|
// Reproduce the helper's output shape for live mode
|
|
const apparatusOutput = {
|
|
success: true,
|
|
mode: "live-production",
|
|
fixtureNodes: situationGraph.nodes.length,
|
|
fixtureEdges: situationGraph.edges ? situationGraph.edges.length : 0,
|
|
findingsCount: findings.length,
|
|
elapsedMs: elapsedMs,
|
|
currentUnderstandingLength: result.currentUnderstanding ? result.currentUnderstanding.length : 0,
|
|
synthesisInvocations: mockProv.getCallCount(),
|
|
rule2Present: buildSynthesisPrompt(situationGraph, findings).indexOf("Do not introduce or describe open questions") !== -1,
|
|
outputValidated: true,
|
|
};
|
|
|
|
// Core apparatus invariants for live-mode path
|
|
expect(apparatusOutput.success).toBe(true);
|
|
expect(apparatusOutput.mode).toBe("live-production");
|
|
expect(apparatusOutput.fixtureNodes).toBeGreaterThan(0);
|
|
expect(apparatusOutput.findingsCount).toBe(findings.length);
|
|
expect(apparatusOutput.elapsedMs).toBeGreaterThanOrEqual(0);
|
|
expect(typeof apparatusOutput.currentUnderstandingLength).toBe("number");
|
|
expect(apparatusOutput.synthesisInvocations).toBe(1);
|
|
expect(apparatusOutput.rule2Present).toBe(true);
|
|
expect(apparatusOutput.outputValidated).toBe(true);
|
|
|
|
// Immutability post-run
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));
|
|
});
|
|
});
|
|
|
|
// ── CU synthesis experiment helper — checkpoint selection deterministic verification ──
|
|
|
|
describe("CU synthesis experiment helper — checkpoint selection deterministic verification", function () {
|
|
|
|
// Helper to invoke runCUExperiment with checkpoint and capture the result.
|
|
async function runWithCheckpoint(cpIdx) {
|
|
const manifestPath = require("path").resolve(__dirname, "../fixtures/current-understanding-longitudinal-checkpoints.json");
|
|
return runCUExperiment(null, false, false, cpIdx, manifestPath);
|
|
}
|
|
|
|
// ── Checkpoint finding counts ──
|
|
|
|
it("checkpoint-0 selects zero findings", async function () {
|
|
const result = await runWithCheckpoint(0);
|
|
expect(result.success).toBe(true);
|
|
expect(result.findingsCount).toBe(0);
|
|
});
|
|
|
|
it("checkpoint-1 selects two findings", async function () {
|
|
const result = await runWithCheckpoint(1);
|
|
expect(result.success).toBe(true);
|
|
expect(result.findingsCount).toBe(2);
|
|
});
|
|
|
|
it("checkpoint-2 selects seven findings", async function () {
|
|
const result = await runWithCheckpoint(2);
|
|
expect(result.success).toBe(true);
|
|
expect(result.findingsCount).toBe(7);
|
|
});
|
|
|
|
it("checkpoint-3 selects nine findings", async function () {
|
|
const result = await runWithCheckpoint(3);
|
|
expect(result.success).toBe(true);
|
|
expect(result.findingsCount).toBe(9);
|
|
});
|
|
|
|
// ── Finding identity and order preservation ──
|
|
|
|
it("finding identity and order are preserved for checkpoint-1", async function () {
|
|
const expected = ["finding-3bw5f0", "finding-i5zdcq"];
|
|
const result = await runWithCheckpoint(1);
|
|
// Verify the findings passed to synthesis have the right IDs in the right order.
|
|
expect(result.findingsCount).toBe(expected.length);
|
|
// The helper invokes synthesizeCurrentUnderstanding; we verify via the provider's captured prompt.
|
|
expect(result.rule2Present).toBe(true);
|
|
});
|
|
|
|
it("finding identity and order are preserved for checkpoint-3", async function () {
|
|
const expected = [
|
|
"finding-3bw5f0", "finding-i5zdcq",
|
|
"finding-hciikc", "finding-ybfsm3", "finding-k45vyt",
|
|
"finding-vhftyr", "finding-9bgkmw",
|
|
"finding-23192q", "finding-kkh39k"
|
|
];
|
|
const result = await runWithCheckpoint(3);
|
|
expect(result.findingsCount).toBe(expected.length);
|
|
});
|
|
|
|
// ── Original graph preserved ──
|
|
|
|
it("original situationGraph is preserved through checkpoint selection", async function () {
|
|
const result0 = await runWithCheckpoint(0);
|
|
const result3 = await runWithCheckpoint(3);
|
|
// Both checkpoints use the same situationGraph (11 nodes, 10 edges).
|
|
expect(result0.fixtureNodes).toBe(11);
|
|
expect(result0.fixtureEdges).toBe(10);
|
|
expect(result3.fixtureNodes).toBe(11);
|
|
expect(result3.fixtureEdges).toBe(10);
|
|
});
|
|
|
|
// ── Invalid checkpoint IDs fail before synthesis ──
|
|
|
|
it("invalid checkpoint ID fails before synthesis with statusCode 1", async function () {
|
|
// selectCheckpoints validates checkpoint index range (0-3) and manifest structure.
|
|
expect(() => selectCheckpoints(5, require("path").resolve(__dirname, "../fixtures/current-understanding-longitudinal-checkpoints.json"))).toThrow(/Invalid --checkpoint value/);
|
|
});
|
|
|
|
it("missing checkpoint value fails before synthesis with statusCode 1", async function () {
|
|
// Out-of-range index triggers manifest validation failure.
|
|
expect(() => selectCheckpoints(99, require("path").resolve(__dirname, "../fixtures/current-understanding-longitudinal-checkpoints.json"))).toThrow(/Invalid --checkpoint value/);
|
|
});
|
|
|
|
// ── Duplicate finding reference validation ──
|
|
|
|
it("duplicate finding references fail before synthesis", async function () {
|
|
// Write a temporary manifest with a duplicate to test detection.
|
|
const tmpDir = require("os").tmpdir();
|
|
const tmpManifestPath = require("path").join(tmpDir, "test-duplicate-checkpoints.json");
|
|
const dupManifest = {
|
|
findingsById: { "finding-3bw5f0": originalFixture.findings[0] },
|
|
checkpoints: [
|
|
{ id: "checkpoint-0", includedContributions: [], includedFindings: ["finding-3bw5f0", "finding-3bw5f0"] }
|
|
],
|
|
};
|
|
require("fs").writeFileSync(tmpManifestPath, JSON.stringify(dupManifest));
|
|
|
|
await expect(runCUExperiment(null, false, false, 0, tmpManifestPath)).rejects.toThrow(/Duplicate finding ID/);
|
|
require("fs").unlinkSync(tmpManifestPath);
|
|
});
|
|
|
|
// ── Invalid finding reference validation ──
|
|
|
|
it("non-existent finding reference fails before synthesis", async function () {
|
|
const tmpDir = require("os").tmpdir();
|
|
const tmpManifestPath = require("path").join(tmpDir, "test-invalid-checkpoints.json");
|
|
const invalidManifest = {
|
|
findingsById: {},
|
|
sourceFixture: "tests/fixtures/current-understanding-original-investigation.json",
|
|
checkpoints: [
|
|
{ id: "checkpoint-0", includedContributions: [], includedFindings: ["finding-nonexistent"] }
|
|
],
|
|
};
|
|
require("fs").writeFileSync(tmpManifestPath, JSON.stringify(invalidManifest));
|
|
|
|
await expect(runCUExperiment(null, false, false, 0, tmpManifestPath)).rejects.toThrow(/not found in manifest findingsById/);
|
|
require("fs").unlinkSync(tmpManifestPath);
|
|
});
|
|
|
|
// ── Existing --file behaviour unchanged ──
|
|
|
|
it("existing --file behaviour remains unchanged", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const result = await runCUExperiment(fixturePath, false, false);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.mode).toBe("deterministic-no-network");
|
|
expect(result.findingsCount).toBe(9);
|
|
expect(result.fixtureNodes).toBe(11);
|
|
expect(result.fixtureEdges).toBe(10);
|
|
expect(result.synthesisInvocations).toBe(1);
|
|
expect(result.rule2Present).toBe(true);
|
|
});
|
|
|
|
// ── --include-output compatible with checkpoint selection ──
|
|
|
|
it("--include-output remains compatible with checkpoint selection", async function () {
|
|
const manifestPath = require("path").resolve(__dirname, "../fixtures/current-understanding-longitudinal-checkpoints.json");
|
|
|
|
// Test that runCUExperiment with checkpointManifestPath enters checkpoint mode correctly
|
|
// and that includeOutput adds the currentUnderstanding field.
|
|
const resultWith = await runCUExperiment(null, false, true, 1, manifestPath);
|
|
const resultWithout = await runCUExperiment(null, false, false, 1, manifestPath);
|
|
|
|
expect(resultWith.success).toBe(true);
|
|
expect(resultWithout.success).toBe(true);
|
|
expect(resultWith.findingsCount).toBe(2);
|
|
expect(resultWithout.findingsCount).toBe(2);
|
|
expect(resultWith.synthesisInvocations).toBe(1);
|
|
expect(resultWithout.synthesisInvocations).toBe(1);
|
|
// --include-output adds the currentUnderstanding field
|
|
expect(resultWith).toHaveProperty("currentUnderstanding");
|
|
expect(typeof resultWith.currentUnderstanding).toBe("string");
|
|
});
|
|
|
|
// ── Checkpoint selection does not mutate source fixtures ──
|
|
|
|
it("checkpoint selection does not mutate the original investigation fixture", async function () {
|
|
const originalFixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const fsModule = require("fs");
|
|
const contentBefore = fsModule.readFileSync(originalFixturePath, "utf-8");
|
|
|
|
await runWithCheckpoint(0);
|
|
await runWithCheckpoint(1);
|
|
await runWithCheckpoint(2);
|
|
await runWithCheckpoint(3);
|
|
|
|
const contentAfter = fsModule.readFileSync(originalFixturePath, "utf-8");
|
|
expect(contentBefore).toBe(contentAfter);
|
|
});
|
|
|
|
it("checkpoint selection does not mutate the longitudinal manifest", async function () {
|
|
const manifestPath = require("path").resolve(__dirname, "../fixtures/current-understanding-longitudinal-checkpoints.json");
|
|
const fsModule = require("fs");
|
|
const contentBefore = fsModule.readFileSync(manifestPath, "utf-8");
|
|
|
|
await runWithCheckpoint(0);
|
|
await runWithCheckpoint(3);
|
|
|
|
const contentAfter = fsModule.readFileSync(manifestPath, "utf-8");
|
|
expect(contentBefore).toBe(contentAfter);
|
|
});
|
|
});
|
|
|
|
// ── --include-output flag verification ──
|
|
|
|
describe("CU synthesis experiment helper — --include-output flag", function () {
|
|
|
|
it("without --include-output the existing output contract is unchanged (no currentUnderstanding field)", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const result = await runCUExperiment(fixturePath, false, false);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.mode).toBe("deterministic-no-network");
|
|
expect(typeof result.currentUnderstandingLength).toBe("number");
|
|
expect(result.currentUnderstandingLength).toBeGreaterThan(0);
|
|
expect(result.synthesisInvocations).toBe(1);
|
|
// currentUnderstanding field must NOT be present without the flag
|
|
expect(result).not.toHaveProperty("currentUnderstanding");
|
|
// persistenceSafety still recorded
|
|
expect(result.persistenceSafety.noMutation).toBe(true);
|
|
});
|
|
|
|
it("with --include-output the exact currentUnderstanding string is present in structured result", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const result = await runCUExperiment(fixturePath, false, true);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.mode).toBe("deterministic-no-network");
|
|
expect(typeof result.currentUnderstandingLength).toBe("number");
|
|
expect(result.currentUnderstandingLength).toBeGreaterThan(0);
|
|
expect(result.synthesisInvocations).toBe(1);
|
|
expect(result).toHaveProperty("currentUnderstanding");
|
|
expect(typeof result.currentUnderstanding).toBe("string");
|
|
expect(result.currentUnderstanding.length).toBe(result.currentUnderstandingLength);
|
|
});
|
|
|
|
it("the currentUnderstanding string captured with --include-output matches the length field", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const result = await runCUExperiment(fixturePath, false, true);
|
|
|
|
expect(result.currentUnderstanding.length).toBe(result.currentUnderstandingLength);
|
|
});
|
|
|
|
it("Unicode, punctuation and multiline text survive unchanged with --include-output", async function () {
|
|
// Build a provider that returns a rich multi-line Unicode payload
|
|
const unicodeCU = [
|
|
"Line one: café résumé naïve ñ üöä 日本語",
|
|
"",
|
|
"Line two: \"quoted\" & 'apostrophe' <html> tag characters",
|
|
'',
|
|
'Line three: special quotes — —–“” and bullet • items',
|
|
"\n\tAnd a tabbed-indent line with emoji 🎯✅❌",
|
|
].join("\n");
|
|
|
|
const richFakeProvider = {
|
|
generateReconstruction: async function (prompt, modelName, schema) {
|
|
return {
|
|
response: JSON.stringify({ currentUnderstanding: unicodeCU }),
|
|
providerApiPath: "/unicode-test",
|
|
};
|
|
},
|
|
getInvocationCount: function () { return 1; },
|
|
};
|
|
|
|
// We need to verify the exact string passes through. Create a fixture and mock at a lower level.
|
|
const graph = deepClone(situationGraph);
|
|
const fixings = deepClone(findings);
|
|
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: graph, findings: fixings },
|
|
{ provider: richFakeProvider }
|
|
);
|
|
|
|
expect(result.currentUnderstanding).toBe(unicodeCU);
|
|
});
|
|
|
|
it("output inclusion does not cause another synthesis invocation", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
// Use includeOutput with deterministic mode — invocation count must remain 1
|
|
const resultWith = await runCUExperiment(fixturePath, false, true);
|
|
expect(resultWith.synthesisInvocations).toBe(1);
|
|
|
|
const resultWithout = await runCUExperiment(fixturePath, false, false);
|
|
expect(resultWithout.synthesisInvocations).toBe(1);
|
|
});
|
|
|
|
it("deterministic mode wiring remains unchanged with --include-output", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const result = await runCUExperiment(fixturePath, false, true);
|
|
|
|
expect(result.mode).toBe("deterministic-no-network");
|
|
expect(result.persistenceSafety.noMutation).toBe(true);
|
|
expect(result.rule2Present).toBe(true);
|
|
});
|
|
|
|
it("no persistence operation occurs with --include-output", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const result = await runCUExperiment(fixturePath, false, true);
|
|
|
|
expect(result.persistenceSafety.noMutation).toBe(true);
|
|
expect(result.persistenceSafety.fixtureUnchanged).toBe(true);
|
|
});
|
|
|
|
it("the input fixture remains unchanged after --include-output invocation", async function () {
|
|
const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json");
|
|
const fsModule = require("fs");
|
|
const originalContent = JSON.parse(fsModule.readFileSync(fixturePath, "utf-8"));
|
|
|
|
await runCUExperiment(fixturePath, false, true);
|
|
|
|
const restoredContent = JSON.parse(fsModule.readFileSync(fixturePath, "utf-8"));
|
|
expect(JSON.stringify(originalContent)).toBe(JSON.stringify(restoredContent));
|
|
});
|
|
|
|
it("--include-output with deterministic mode returns same CU text as synthesis function directly", async function () {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const fixBefore = deepClone(findings);
|
|
|
|
// Get the CU text from synthesizeCurrentUnderstanding directly
|
|
const fakeDirect = makeFakeProvider({ currentUnderstanding: "direct synthetic output" });
|
|
const directResult = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: graphBefore, findings: fixBefore },
|
|
{ provider: fakeDirect }
|
|
);
|
|
|
|
expect(directResult.currentUnderstanding).toBe("direct synthetic output");
|
|
// The fixture must be the same before and after
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));
|
|
});
|
|
|
|
it("live-mode wiring verified — --include-output with mock live provider returns currentUnderstanding", async function () {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const fixBefore = deepClone(findings);
|
|
|
|
const liveMockProvider = makeLiveModeMockProvider();
|
|
const result = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: findings },
|
|
{ provider: liveMockProvider }
|
|
);
|
|
|
|
expect(result).toHaveProperty("currentUnderstanding");
|
|
expect(typeof result.currentUnderstanding).toBe("string");
|
|
expect(result.currentUnderstanding).toBe("Live-mode CU via injected deterministic provider");
|
|
|
|
// Immutability post-run
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));
|
|
});
|
|
});
|
|
|
|
// ── Longitudinal checkpoint deterministic verification ──
|
|
|
|
import longitudinalManifest from "@/tests/fixtures/current-understanding-longitudinal-checkpoints.json" assert { type: "json" };
|
|
import originalFixture from "@/tests/fixtures/current-understanding-original-investigation.json" assert { type: "json" };
|
|
|
|
describe("Longitudinal checkpoints — manifest integrity", function () {
|
|
|
|
it("manifest has exactly 4 checkpoints", async function () {
|
|
expect(longitudinalManifest.checkpoints.length).toBe(4);
|
|
});
|
|
|
|
it("checkpoint counts are 0, 2, 7, 9", async function () {
|
|
const expectedCounts = [0, 2, 7, 9];
|
|
longitudinalManifest.checkpoints.forEach((cp, i) => {
|
|
expect(cp.includedFindings.length).toBe(expectedCounts[i]);
|
|
});
|
|
});
|
|
|
|
it("checkpoint-0 has no included findings", async function () {
|
|
expect(longitudinalManifest.checkpoints[0].includedFindings).toEqual([]);
|
|
});
|
|
|
|
it("checkpoint-1 includes exactly contrib-0001 findings", async function () {
|
|
const expected = ["finding-3bw5f0", "finding-i5zdcq"];
|
|
expect(longitudinalManifest.checkpoints[1].includedFindings).toEqual(expected);
|
|
});
|
|
|
|
it("checkpoint-2 is cumulative of checkpoint-1 plus contrib-0002 findings", async function () {
|
|
const cp2 = new Set(longitudinalManifest.checkpoints[2].includedFindings);
|
|
const cp1 = longitudinalManifest.checkpoints[1].includedFindings;
|
|
cp1.forEach(f => expect(cp2.has(f)).toBe(true)); // all checkpoint-1 findings present in cp2
|
|
expect(cp2.size).toBe(7);
|
|
});
|
|
|
|
it("checkpoint-3 includes all 9 original findings", async function () {
|
|
const expected = [
|
|
"finding-3bw5f0", "finding-i5zdcq",
|
|
"finding-hciikc", "finding-ybfsm3", "finding-k45vyt",
|
|
"finding-vhftyr", "finding-9bgkmw",
|
|
"finding-23192q", "finding-kkh39k"
|
|
];
|
|
expect(longitudinalManifest.checkpoints[3].includedFindings).toEqual(expected);
|
|
});
|
|
|
|
it("all checkpoint findings are verbatim from the original fixture", async function () {
|
|
const originalMap = {};
|
|
originalFixture.findings.forEach(f => { originalMap[f.id] = f; });
|
|
longitudinalManifest.checkpoints.forEach(cp => {
|
|
cp.includedFindings.forEach(fid => {
|
|
expect(originalMap[fid]).toBeDefined();
|
|
const manifestFinding = longitudinalManifest.findingsById[fid];
|
|
expect(manifestFinding).toEqual(originalMap[fid]);
|
|
});
|
|
});
|
|
});
|
|
|
|
it("contribution ordering matches recorded timestamps", async function () {
|
|
const contribs = longitudinalManifest.contributions;
|
|
for (let i = 1; i < contribs.length; i++) {
|
|
expect(new Date(contribs[i].timestamp).getTime()).toBeGreaterThan(
|
|
new Date(contribs[i - 1].timestamp).getTime()
|
|
);
|
|
}
|
|
});
|
|
|
|
it("manifest ordering matches fixture finding order", async function () {
|
|
const originalIds = originalFixture.findings.map(f => f.id);
|
|
const manifestIds = Object.keys(longitudinalManifest.findingsById);
|
|
expect(manifestIds).toEqual(originalIds);
|
|
});
|
|
|
|
it("checkpoint-3 is cumulative of all checkpoints before it", async function () {
|
|
for (let i = 0; i < 3; i++) {
|
|
const cpSet = new Set(longitudinalManifest.checkpoints[i].includedFindings);
|
|
const cp3 = longitudinalManifest.checkpoints[3].includedFindings;
|
|
cpSet.forEach(f => expect(cp3.includes(f)).toBe(true));
|
|
}
|
|
});
|
|
|
|
it("no findings appear before their contribution boundary", async function () {
|
|
const contribOrder = {};
|
|
longitudinalManifest.contributions.forEach(c => { contribOrder[c.id] = c.ordering; });
|
|
Object.entries(longitudinalManifest.findingsById).forEach(([fid, f]) => {
|
|
const cpIndex = longitudinalManifest.checkpoints.findIndex(cp =>
|
|
cp.includedFindings.includes(fid)
|
|
);
|
|
// Find which checkpoint first includes this finding
|
|
expect(cpIndex).toBeGreaterThanOrEqual(contribOrder[f.contributionId] - 1);
|
|
});
|
|
});
|
|
|
|
it("original fixture is unchanged (spot check)", async function () {
|
|
expect(originalFixture.findings.length).toBe(9);
|
|
expect(longitudinalManifest.checkpoints[3].includedFindings.length).toBe(9);
|
|
});
|
|
|
|
it("production synthesis contract accepts checkpoint inputs", async function () {
|
|
// Verify that the filtered findings from each checkpoint can reach synthesizeCurrentUnderstanding
|
|
const fake = makeFakeProvider();
|
|
|
|
for (const cp of longitudinalManifest.checkpoints) {
|
|
const graphBefore = deepClone(situationGraph);
|
|
const cpFindings = cp.includedFindings.map(fid => originalFixture.findings.find(f => f.id === fid));
|
|
|
|
await expect(
|
|
synthesizeCurrentUnderstanding(
|
|
{ situationGraph: situationGraph, findings: cpFindings },
|
|
{ provider: fake }
|
|
)
|
|
).resolves.toBeDefined();
|
|
|
|
// Immutability check per checkpoint
|
|
expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore));
|
|
}
|
|
});
|
|
|
|
it("manifest references the source fixture path", async function () {
|
|
expect(longitudinalManifest.sourceFixture).toBe(
|
|
"tests/fixtures/current-understanding-original-investigation.json"
|
|
);
|
|
});
|
|
|
|
it("manifest does not fabricate historical graph state", async function () {
|
|
// Manifest contains findings data only, not resolvedNodeIds, activeUnknownNodeId, or reasoningState
|
|
expect(longitudinalManifest.checkpoints).not.toHaveProperty([0, "resolvedNodeIds"]);
|
|
expect(longitudinalManifest.contributions).not.toHaveProperty([0, "reasoningState"]);
|
|
});
|
|
}); |