feat(confidence-engine): add investigation overview synthesis seam
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Investigation Overview synthesis API route.
|
||||
*
|
||||
* Route: POST /api/cases/overview
|
||||
*
|
||||
* Thin route pattern — no overview business logic here.
|
||||
*/
|
||||
|
||||
import { getProvider } from "@/lib/llm/provider.js";
|
||||
import { synthesizeInvestigationOverview } from "@/lib/graph/investigation-overview-synthesis.js";
|
||||
|
||||
export async function POST(request) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
|
||||
if (!body || typeof body !== "object") {
|
||||
return Response.json(
|
||||
{ success: false, stage: "request_validation", error: "Invalid request body" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { situationGraph, findings, plausibleInterpretations } = body;
|
||||
|
||||
if (!situationGraph) {
|
||||
return Response.json(
|
||||
{ success: false, stage: "request_validation", error: "Missing situationGraph" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const result = await synthesizeInvestigationOverview(
|
||||
{ situationGraph, findings, plausibleInterpretations },
|
||||
{
|
||||
provider: getProvider(),
|
||||
modelName: process.env.OLLAMA_MODEL ?? null,
|
||||
}
|
||||
);
|
||||
|
||||
return Response.json(
|
||||
{ success: true, understanding: result.understanding, plausibleInterpretations: result.plausibleInterpretations },
|
||||
{ status: 200 }
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) {
|
||||
return Response.json(
|
||||
{ success: false, stage: "request_validation", error: "Invalid JSON request body" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (error.statusCode) {
|
||||
return Response.json(
|
||||
{
|
||||
success: false,
|
||||
stage: error.statusCode === 400 ? "request_validation" : "provider",
|
||||
error: error.message ?? "Overview synthesis failed",
|
||||
},
|
||||
{ status: error.statusCode }
|
||||
);
|
||||
}
|
||||
|
||||
return Response.json(
|
||||
{ success: false, stage: "internal", error: "Internal server error" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
+27
-1
@@ -354,7 +354,33 @@ A fresh unanswered Question B displayed stale focused-investigation content from
|
||||
- Reasoning, prompts, providers, episode preparation unchanged
|
||||
- Zero-Open-Questions milestone and focused-presentation ownership (v0.52) remain unchanged
|
||||
|
||||
### Open defects
|
||||
### v0.54 — Investigation overview synthesis apparatus established
|
||||
|
||||
**Objective:** Establish the smallest reusable seam for a later bounded live experiment answering whether one additional synthesis call can produce a more useful investigation overview than Current Understanding, while keeping established understanding and plausible interpretations epistemically separate.
|
||||
|
||||
**Apparatus delivered (no UI integration, no live semantic experiment):**
|
||||
|
||||
- **Domain function:** `synthesizeInvestigationOverview()` in `lib/graph/investigation-overview-synthesis.js`
|
||||
- Accepts `{ situationGraph, findings, plausibleInterpretations }`
|
||||
- Output contract: `{ understanding, plausibleInterpretations }` — two structurally distinct string fields
|
||||
- Uses Zod-safeParse validation rejecting any recommendation/decision/confidenceScore/nextAction/priority/readiness leakage
|
||||
- Epistemic boundary rules: established evidence never promoted to interpretation; interpretations never promoted to understanding
|
||||
|
||||
- **Route:** `POST /api/cases/overview` (thin route, parallel to existing `/api/cases/synthesis`)
|
||||
|
||||
- **Deterministic tests:** `tests/graph/investigation-overview-synthesis.test.js` — 49 tests covering epistemic boundary integrity, evidence exclusion, interpretation separation, output contract, and full seam
|
||||
|
||||
**Plausible interpretation ownership (discovered):**
|
||||
- Canonical source: `reconstruction.plausibleInterpretations` in `lib/reconstruction/schema.js`
|
||||
- Schema: `{ id, description, supportingEvidenceIds[], assumptionsRequired[], confidence }`
|
||||
- Written to graph as `"assumption"` nodes with `status: "provisional"` via `lib/graph/builder.js`
|
||||
- Presentation-derived (UI renders from reconstructed provisional assumption nodes); canonical state is the reconstruction payload
|
||||
|
||||
**Existing CU synthesis unchanged:** `synthesizeCurrentUnderstanding()`, `buildGraphEvidenceProjection()`, `filterEligibleFindings()` untouched.
|
||||
|
||||
> v0.54 overview synthesis apparatus established; no live semantic experiment and no UI integration performed yet.
|
||||
|
||||
## Open defects
|
||||
|
||||
- Empty Done `no_episodic_content`: choosing Done without episodic content can produce `{ success: false, stage: "preparation", error: "no_episodic_content" }` — separate future increment (empty-Done orchestration guard now prevents the 400 in practice by skipping episode processing entirely)
|
||||
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
/**
|
||||
* Investigation Overview synthesis seam — standalone domain function.
|
||||
*
|
||||
* Purpose: produce a structurally distinct two-part overview that keeps
|
||||
* (A) evidence-backed understanding and
|
||||
* (B) remaining plausible interpretations
|
||||
* epistemically separate.
|
||||
*
|
||||
* Input contract:
|
||||
* { situationGraph, findings, plausibleInterpretations }
|
||||
*
|
||||
* Output contract:
|
||||
* {
|
||||
* "understanding": "evidence-backed synthesis",
|
||||
* "plausibleInterpretations": "qualified synthesis of remaining interpretations"
|
||||
* }
|
||||
*
|
||||
* Does NOT produce: recommendation, decision, confidence score, next action, priority, readiness.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
import { getProvider } from "../llm/provider.js";
|
||||
import { filterEligibleFindings } from "./current-understanding-synthesis.js";
|
||||
|
||||
export { filterEligibleFindings };
|
||||
|
||||
// ── Overview-specific output validation schema ────────────────
|
||||
|
||||
const overviewResponseSchema = z.object({
|
||||
understanding: z.string().min(1),
|
||||
plausibleInterpretations: z.string().min(1),
|
||||
});
|
||||
|
||||
const FORBIDDEN_FIELD_NAMES = new Set([
|
||||
"recommendation",
|
||||
"decision",
|
||||
"confidenceScore",
|
||||
"nextAction",
|
||||
"priority",
|
||||
"readiness",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Validate that the raw overview response has exactly two semantic fields:
|
||||
* - understanding (string)
|
||||
* - plausibleInterpretations (string)
|
||||
* and no decision/recommendation/priority/readiness/next-action leakage.
|
||||
*/
|
||||
export function validateOverviewResponse(raw) {
|
||||
if (raw == null) {
|
||||
return { valid: false, reason: "Provider returned null/undefined" };
|
||||
}
|
||||
|
||||
let parsed;
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
return { valid: false, reason: "Provider output is not valid JSON" };
|
||||
}
|
||||
} else if (typeof raw === "object") {
|
||||
parsed = raw;
|
||||
} else {
|
||||
return { valid: false, reason: "Provider output has unexpected type" };
|
||||
}
|
||||
|
||||
// Reject any forbidden epistemic fields
|
||||
for (const key of Object.keys(parsed)) {
|
||||
if (FORBIDDEN_FIELD_NAMES.has(key)) {
|
||||
return { valid: false, reason: `forbidden_field: ${key}` };
|
||||
}
|
||||
}
|
||||
|
||||
const result = overviewResponseSchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
return { valid: false, reason: "Missing or invalid required fields" };
|
||||
}
|
||||
|
||||
return { valid: true, data: result.data };
|
||||
}
|
||||
|
||||
// ── Overview-specific prompt construction ─────────────────────
|
||||
|
||||
const KNOWN_SUPPORTED_STATUSES = new Set(["known", "supported"]);
|
||||
|
||||
function safeDesc(value) {
|
||||
return (value && typeof value === "string") ? value : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the overview synthesis prompt from SituationGraph, eligible Findings,
|
||||
* and plausible interpretations.
|
||||
*
|
||||
* Produces three evidence sections:
|
||||
* 1. Evidence-backed understanding inputs (known + supported nodes + eligible Findings)
|
||||
* 2. Plausible interpretations inputs (kept separate from evidence)
|
||||
* 3. Epistemic boundary rules
|
||||
*/
|
||||
export function buildOverviewSynthesisPrompt(situationGraph, findings, plausibleInterpretations) {
|
||||
// Evidence-backed projection: reuse the existing known+supported logic
|
||||
const knownNodes = (situationGraph.nodes ?? [])
|
||||
.filter((n) => KNOWN_SUPPORTED_STATUSES.has(n.status))
|
||||
.filter((n) => n.status === "known")
|
||||
.map((n) => ({
|
||||
kind: n.kind ?? null,
|
||||
label: safeDesc(n.label),
|
||||
description: safeDesc(n.description),
|
||||
value: n.value ?? null,
|
||||
unit: n.unit ?? null,
|
||||
status: n.status ?? null,
|
||||
}));
|
||||
|
||||
const supportedNodes = (situationGraph.nodes ?? [])
|
||||
.filter((n) => KNOWN_SUPPORTED_STATUSES.has(n.status))
|
||||
.filter((n) => n.status !== "known")
|
||||
.map((n) => ({
|
||||
kind: n.kind ?? null,
|
||||
label: safeDesc(n.label),
|
||||
description: safeDesc(n.description),
|
||||
value: n.value ?? null,
|
||||
unit: n.unit ?? null,
|
||||
status: n.status ?? null,
|
||||
}));
|
||||
|
||||
const centralStatement = safeDesc(situationGraph.centralStatement) || "";
|
||||
|
||||
// Eligible Findings (reuse existing filter)
|
||||
const eligibleFindings = filterEligibleFindings(findings);
|
||||
const agreedFindings = eligibleFindings.filter((f) => f.userDisposition === "agree");
|
||||
const workingFindings = eligibleFindings.filter((f) => f.userDisposition === null);
|
||||
|
||||
// Format nodes for prompt display
|
||||
function formatNodes(nodes, title) {
|
||||
if (!nodes || nodes.length === 0) return "";
|
||||
return nodes.map(
|
||||
(n) => ` ${title}: kind=${n.kind}, label="${n.label}", value=${n.value ? n.value + (n.unit ? " (" + n.unit + ")" : "") : null} — ${n.description ?? "(no description)"} [${n.status}]`
|
||||
).join("\n");
|
||||
}
|
||||
|
||||
const knownSection = formatNodes(knownNodes, "Known");
|
||||
const supportedSection = formatNodes(supportedNodes, "Supported");
|
||||
|
||||
// Format plausible interpretations (kept separate from evidence)
|
||||
const interpSections = [];
|
||||
if (plausibleInterpretations && Array.isArray(plausibleInterpretations)) {
|
||||
for (const interp of plausibleInterpretations) {
|
||||
interpSections.push({
|
||||
id: interp.id ?? null,
|
||||
description: safeDesc(interp.description) || "Unlabelled interpretation",
|
||||
confidence: interp.confidence ?? "unknown",
|
||||
supportingEvidenceIds: interp.supportingEvidenceIds ?? [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const findingsSections = [];
|
||||
if (agreedFindings.length > 0) {
|
||||
findingsSections.push({
|
||||
label: "Confirmed Evidence",
|
||||
items: agreedFindings.map((f) => ({ proposition: f.proposition, id: f.id ?? null })),
|
||||
});
|
||||
}
|
||||
if (workingFindings.length > 0) {
|
||||
findingsSections.push({
|
||||
label: "Working Premises",
|
||||
items: workingFindings.map((f) => ({ proposition: f.proposition, id: f.id ?? null })),
|
||||
});
|
||||
}
|
||||
|
||||
const prompt = `You are producing an investigation overview with two structurally distinct sections.
|
||||
|
||||
Situation Framing:
|
||||
${centralStatement ? " Central Statement: " + centralStatement : "(none)"}
|
||||
|
||||
=== SECTION A INPUTS — Evidence-Backed Understanding ===
|
||||
|
||||
Provider-Active Evidence:
|
||||
Known Facts:${knownSection || " (none)"}
|
||||
Supported Inferences:${supportedSection || " (none)"}
|
||||
|
||||
Eligible Findings:
|
||||
${findingsSections.length > 0 ? JSON.stringify(findingsSections, null, 2) : "(none)"}
|
||||
|
||||
=== SECTION B INPUTS — Plausible Interpretations (NOT evidence-backed) ===
|
||||
|
||||
Plausible Interpretations:
|
||||
${interpSections.length > 0 ? JSON.stringify(interpSections, null, 2) : "(none)"}
|
||||
|
||||
=== EPISTEMIC BOUNDARY RULES ===
|
||||
|
||||
1. Section A (understanding) MUST contain only established or supported understanding from the evidence in SECTION A INPUTS above.
|
||||
2. Section A MUST NOT include open questions, unresolved uncertainties, assumptions, provisional hypotheses, speculative explanations, or future investigation needs.
|
||||
3. Plausible interpretations from SECTION B INPUTS MUST remain explicitly qualified as interpretations — never promoted into Section A (understanding).
|
||||
4. Plausible interpretations must not be presented as established evidence or confirmed facts.
|
||||
5. Produce exactly ONE coherent narrative paragraph for "understanding" from SECTION A inputs only.
|
||||
6. Produce exactly ONE coherent narrative paragraph for "plausibleInterpretations" from SECTION B inputs only. Each interpretation should be clearly qualified as an interpretation.
|
||||
7. Do NOT introduce any new facts not present in the supplied evidence.
|
||||
8. Return ONLY a JSON object with this exact structure:
|
||||
{"understanding": "...", "plausibleInterpretations": "..."}
|
||||
9. Neither field may contain recommendations, decisions, confidence scores, next actions, priorities, or readiness assessments.
|
||||
10. This is a FRESH synthesis — do NOT treat any previous overview or Current Understanding as input.
|
||||
|
||||
Both fields must be non-empty strings.`;
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
// ── Overview domain function ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* Synthesize an investigation overview with structurally distinct sections:
|
||||
* - understanding: evidence-backed synthesis
|
||||
* - plausibleInterpretations: qualified remaining interpretations
|
||||
*
|
||||
* @param {{ situationGraph, findings, plausibleInterpretations }} params
|
||||
* @param {{ provider, modelName }} deps
|
||||
* @returns {Promise<{ understanding: string, plausibleInterpretations: string }>}
|
||||
*/
|
||||
export async function synthesizeInvestigationOverview(params, deps) {
|
||||
const { situationGraph, findings = [], plausibleInterpretations = [] } = params;
|
||||
|
||||
if (!situationGraph || typeof situationGraph !== "object") {
|
||||
throw new Error("situationGraph is required");
|
||||
}
|
||||
if (!Array.isArray(findings)) {
|
||||
throw new Error("findings must be an array");
|
||||
}
|
||||
if (!Array.isArray(plausibleInterpretations)) {
|
||||
throw new Error("plausibleInterpretations must be an array");
|
||||
}
|
||||
|
||||
// Provider acquisition (reuse existing pattern)
|
||||
const provider = deps?.provider ?? getProvider();
|
||||
const modelName = deps?.modelName ?? process.env.OLLAMA_MODEL;
|
||||
|
||||
if (!provider || typeof provider.generateReconstruction !== "function") {
|
||||
throw new Error("Invalid dependency: provider must have generateReconstruction");
|
||||
}
|
||||
|
||||
// Build overview-specific prompt (keeps evidence/interpretation separate)
|
||||
const prompt = buildOverviewSynthesisPrompt(situationGraph, findings, plausibleInterpretations);
|
||||
|
||||
let rawResponse;
|
||||
try {
|
||||
rawResponse = await provider.generateReconstruction(
|
||||
prompt,
|
||||
modelName,
|
||||
);
|
||||
} catch (err) {
|
||||
throw new Error(`Overview synthesis provider call failed: ${err.message}`);
|
||||
}
|
||||
|
||||
// Validate against overview-specific contract
|
||||
const validated = validateOverviewResponse(rawResponse);
|
||||
if (!validated.valid) {
|
||||
throw new Error(`Overview synthesis validation failed: ${validated.reason}`);
|
||||
}
|
||||
|
||||
return validated.data;
|
||||
}
|
||||
@@ -0,0 +1,462 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
filterEligibleFindings,
|
||||
buildOverviewSynthesisPrompt,
|
||||
synthesizeInvestigationOverview,
|
||||
validateOverviewResponse,
|
||||
} from "@/lib/graph/investigation-overview-synthesis.js";
|
||||
|
||||
// ── Fixtures ────────────────────────────────────────────────
|
||||
|
||||
const graph = {
|
||||
centralStatement: "Revenue declined while churn increased.",
|
||||
nodes: [
|
||||
{ id: "n1", label: "Revenue down 12%", description: "Primary metric baseline", kind: "metric", status: "known", confidence: "high", value: 12, unit: "%" },
|
||||
{ id: "n2", label: "Customer churn up 8%", description: "Secondary metric baseline", kind: "metric", status: "supported", confidence: "medium", value: 8, unit: "%" },
|
||||
{ id: "n3", label: "Open question about baseline period", description: "Unknown", kind: "unknown", status: "unknown", confidence: "low" },
|
||||
{ id: "n4", label: "Resolved time period question", description: "Resolved answer", kind: "question", status: "resolved", confidence: "high" },
|
||||
{ id: "n5", label: "Provisional hypothesis about market conditions", description: "Not yet tested", kind: "hypothesis", status: "provisional", confidence: "low" },
|
||||
],
|
||||
edges: [{ fromNodeId: "n1", toNodeId: "n2", relationship: "correlates_with" }],
|
||||
activeUnknownNodeId: "n3",
|
||||
resolvedNodeIds: ["n4"],
|
||||
currentSummary: "Sentinel: must not appear",
|
||||
};
|
||||
|
||||
const eligibleFindings = [
|
||||
{ id: "f1", proposition: "Revenue decline matches sector trend.", userDisposition: "agree" },
|
||||
{ id: "f2", proposition: "Churn increase correlates with pricing change date.", userDisposition: null },
|
||||
{ id: "f3", proposition: "This finding is not relevant.", userDisposition: "not_relevant" },
|
||||
{ id: "f4", proposition: "Rejected finding.", evaluation: "rejected", userDisposition: null },
|
||||
];
|
||||
|
||||
const plausibleInterps = [
|
||||
{ id: "p1", description: "Market-wide downturn could explain revenue decline.", confidence: "medium", supportingEvidenceIds: ["n1"] },
|
||||
{ id: "p2", description: "Product feature regression might cause churn increase.", confidence: "low", supportingEvidenceIds: ["n2"] },
|
||||
];
|
||||
|
||||
const fakeProvider = (defaultResponse) => ({
|
||||
generateReconstruction: vi.fn(() => {
|
||||
return typeof defaultResponse === "function"
|
||||
? defaultResponse()
|
||||
: JSON.stringify({
|
||||
understanding: "Synthesized understanding from evidence.",
|
||||
plausibleInterpretations: "Remaining plausible explanations.",
|
||||
});
|
||||
}),
|
||||
});
|
||||
|
||||
// ── 1. Supported evidence is available to the understanding synthesis ───────────────
|
||||
|
||||
describe("investigation overview — evidence availability", () => {
|
||||
it("known node content flows to understanding section input", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Revenue down 12%");
|
||||
expect(prompt).toContain("[known]");
|
||||
expect(prompt).toContain("Known Facts:");
|
||||
});
|
||||
|
||||
it("supported node content flows to understanding section input", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Customer churn up 8%");
|
||||
expect(prompt).toContain("[supported]");
|
||||
expect(prompt).toContain("Supported Inferences:");
|
||||
});
|
||||
|
||||
it("eligible agreed Finding flows to understanding section input", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Revenue decline matches sector trend.");
|
||||
expect(prompt).toContain("Confirmed Evidence");
|
||||
});
|
||||
|
||||
it("eligible working premise Finding flows to understanding section input", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Churn increase correlates with pricing change date.");
|
||||
expect(prompt).toContain("Working Premises");
|
||||
});
|
||||
|
||||
it("plausible interpretations flow to separate section input", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Market-wide downturn could explain revenue decline.");
|
||||
expect(prompt).toContain("Plausible Interpretations:");
|
||||
expect(prompt).toContain("SECTION B INPUTS");
|
||||
});
|
||||
|
||||
it("centralStatement flows as framing context", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Revenue declined while churn increased.");
|
||||
});
|
||||
});
|
||||
|
||||
// ── 2. Unresolved graph material is excluded ───────────────────────
|
||||
|
||||
describe("investigation overview — unresolved material excluded", () => {
|
||||
it("unknown nodes do NOT appear in the overview prompt", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).not.toContain("Open question about baseline period");
|
||||
expect(prompt).not.toContain('status":"unknown"');
|
||||
});
|
||||
|
||||
it("provisional nodes do NOT appear in the overview prompt", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).not.toContain("Provisional hypothesis about market conditions");
|
||||
expect(prompt).not.toContain('status":"provisional"');
|
||||
});
|
||||
|
||||
it("resolved node text does NOT appear in the overview prompt", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).not.toContain("Resolved time period question");
|
||||
expect(prompt).not.toContain('status":"resolved"');
|
||||
});
|
||||
|
||||
it("control fields excluded from overview prompt", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).not.toContain("currentSummary");
|
||||
expect(prompt).not.toContain("Sentinel: must not appear");
|
||||
expect(prompt).not.toContain("reasoningState");
|
||||
expect(prompt).not.toContain("activeUnknownNodeId");
|
||||
expect(prompt).not.toContain("resolvedNodeIds");
|
||||
});
|
||||
|
||||
it("edges do NOT appear in the overview prompt", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).not.toContain("correlates_with");
|
||||
});
|
||||
|
||||
it("ineligible findings (not_relevant + rejected) excluded from evidence input", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).not.toContain("This finding is not relevant.");
|
||||
expect(prompt).not.toContain("Rejected finding.");
|
||||
});
|
||||
});
|
||||
|
||||
// ── 3. Plausible interpretations remain separately represented ───────────────
|
||||
|
||||
describe("investigation overview — interpretation separation", () => {
|
||||
it("plausible interpretations appear only in SECTION B, not SECTION A", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
// They should appear under Plausible Interpretations (section B)
|
||||
expect(prompt).toContain("Plausible Interpretations:");
|
||||
// And also appear once in the document (in section B only)
|
||||
const lines = prompt.split("\n");
|
||||
let foundInSectionA = false;
|
||||
let foundInSectionB = false;
|
||||
let currentSection = null;
|
||||
for (const line of lines) {
|
||||
if (line.includes("SECTION A")) currentSection = "a";
|
||||
if (line.includes("SECTION B")) currentSection = "b";
|
||||
if (currentSection === "a" && line.includes("Product feature regression")) foundInSectionA = true;
|
||||
if (currentSection === "b" && line.includes("Product feature regression")) foundInSectionB = true;
|
||||
}
|
||||
expect(foundInSectionA).toBe(false);
|
||||
expect(foundInSectionB).toBe(true);
|
||||
});
|
||||
|
||||
it("empty plausible interpretations handled gracefully", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, []);
|
||||
expect(prompt).toContain("SECTION B INPUTS");
|
||||
// When empty, "(none)" indicates no interpretations — acceptable behavior
|
||||
expect(prompt).toContain("SECTION B INPUTS — Plausible Interpretations");
|
||||
});
|
||||
});
|
||||
|
||||
// ── 4. Plausible interpretations NOT inserted into evidence-backed projection ───────────────
|
||||
|
||||
describe("investigation overview — epistemic boundary integrity", () => {
|
||||
it("plausible interpretations are never included in the known+supported node projection", () => {
|
||||
// Create a graph without any plausible interpretation nodes (which would be status "provisional")
|
||||
const cleanGraph = {
|
||||
centralStatement: "Test scenario.",
|
||||
nodes: [
|
||||
{ id: "a1", label: "Confirmed fact", description: "Evidence", kind: "metric", status: "known" },
|
||||
{ id: "a2", label: "Supported inference", description: "Evidence", kind: "metric", status: "supported" },
|
||||
],
|
||||
};
|
||||
// Even when plausibleInterps are passed, they must not enter the evidence section
|
||||
const prompt = buildOverviewSynthesisPrompt(cleanGraph, [], [{ id: "x1", description: "A plausible interp with its own id and description.", confidence: "low" }]);
|
||||
|
||||
// SECTION A has only a1/a2 content — no interpretation leakage
|
||||
expect(prompt).toContain("Confirmed fact");
|
||||
expect(prompt).toContain("[known]");
|
||||
});
|
||||
|
||||
it("passing plausibleInterpretations with no graph evidence still produces valid prompt", () => {
|
||||
const minimalGraph = { centralStatement: "Minimal." };
|
||||
const prompt = buildOverviewSynthesisPrompt(minimalGraph, [], plausibleInterps);
|
||||
expect(prompt).toContain("SECTION A INPUTS");
|
||||
expect(prompt).toContain("SECTION B INPUTS");
|
||||
expect(prompt).not.toContain("[known]");
|
||||
expect(prompt).not.toContain("[supported]");
|
||||
});
|
||||
|
||||
it("prompt contains explicit epistemic boundary rules", () => {
|
||||
const prompt = buildOverviewSynthesisPrompt(graph, eligibleFindings, plausibleInterps);
|
||||
expect(prompt).toContain("Section A (understanding) MUST contain only established or supported understanding");
|
||||
expect(prompt).toContain("never promoted into Section A");
|
||||
expect(prompt).toContain("Do NOT introduce any new facts");
|
||||
// SECTION B must also be present for interpretation content
|
||||
expect(prompt).toContain("SECTION B INPUTS");
|
||||
});
|
||||
});
|
||||
|
||||
// ── 5. Output contract — structurally distinct fields ───────────────
|
||||
|
||||
describe("investigation overview — output validation contract", () => {
|
||||
it("valid two-field response accepted", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y remains plausible.",
|
||||
});
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.data.understanding).toBe("We understand X.");
|
||||
expect(result.data.plausibleInterpretations).toBe("Y remains plausible.");
|
||||
});
|
||||
|
||||
it("valid JSON string response accepted", () => {
|
||||
const result = validateOverviewResponse(JSON.stringify({ understanding: "parsed", plausibleInterpretations: "also parsed" }));
|
||||
expect(result.valid).toBe(true);
|
||||
});
|
||||
|
||||
it("missing understanding field rejected", () => {
|
||||
const result = validateOverviewResponse({ plausibleInterpretations: "Only one field" });
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("missing plausibleInterpretations field rejected", () => {
|
||||
const result = validateOverviewResponse({ understanding: "Only one field" });
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("empty string understanding rejected", () => {
|
||||
const result = validateOverviewResponse({ understanding: "", plausibleInterpretations: "something" });
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("empty string plausibleInterpretations rejected", () => {
|
||||
const result = validateOverviewResponse({ understanding: "something", plausibleInterpretations: "" });
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("malformed JSON rejected", () => {
|
||||
const result = validateOverviewResponse("not json [[[");
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("null input rejected", () => {
|
||||
expect(validateOverviewResponse(null).valid).toBe(false);
|
||||
});
|
||||
|
||||
it("undefined input rejected", () => {
|
||||
expect(validateOverviewResponse(undefined).valid).toBe(false);
|
||||
});
|
||||
|
||||
it("non-object input rejected", () => {
|
||||
expect(validateOverviewResponse("just a string").valid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 6. No decision/recommendation/readiness fields allowed ───────────────
|
||||
|
||||
describe("investigation overview — no forbidden epistemic leakage", () => {
|
||||
it("recommendation field rejected by validator", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y is plausible.",
|
||||
recommendation: "Do Y next.",
|
||||
});
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.reason).toContain("forbidden_field");
|
||||
});
|
||||
|
||||
it("decision field rejected by validator", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y is plausible.",
|
||||
decision: "Proceed with Y.",
|
||||
});
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("confidenceScore field rejected by validator", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y is plausible.",
|
||||
confidenceScore: 0.8,
|
||||
});
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("nextAction field rejected by validator", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y is plausible.",
|
||||
nextAction: "Investigate Y.",
|
||||
});
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("priority field rejected by validator", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y is plausible.",
|
||||
priority: "high",
|
||||
});
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it("readiness field rejected by validator", () => {
|
||||
const result = validateOverviewResponse({
|
||||
understanding: "We understand X.",
|
||||
plausibleInterpretations: "Y is plausible.",
|
||||
readiness: "ready",
|
||||
});
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Domain function seam tests (full overview synthesis) ───────────────
|
||||
|
||||
describe("synthesizeInvestigationOverview — full seam", () => {
|
||||
it("produces structured overview output from valid inputs", async () => {
|
||||
const fake = fakeProvider();
|
||||
const result = await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: eligibleFindings, plausibleInterpretations: plausibleInterps },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result).toHaveProperty("understanding");
|
||||
expect(result).toHaveProperty("plausibleInterpretations");
|
||||
expect(typeof result.understanding).toBe("string");
|
||||
expect(typeof result.plausibleInterpretations).toBe("string");
|
||||
expect(fake.generateReconstruction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("provider receives prompt with evidence-authority boundary rules", async () => {
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: eligibleFindings, plausibleInterpretations: plausibleInterps },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).toContain("EPISTEMIC BOUNDARY RULES");
|
||||
expect(prompt).toContain("SECTION A INPUTS");
|
||||
expect(prompt).toContain("SECTION B INPUTS");
|
||||
expect(prompt).toContain("never promoted into Section A");
|
||||
});
|
||||
|
||||
it("unknown nodes do NOT appear in provider-visible prompt", async () => {
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: eligibleFindings, plausibleInterpretations: plausibleInterps },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).not.toContain("Open question about baseline period");
|
||||
});
|
||||
|
||||
it("provisional nodes do NOT appear in provider-visible prompt", async () => {
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: eligibleFindings, plausibleInterpretations: plausibleInterps },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).not.toContain("Provisional hypothesis");
|
||||
});
|
||||
|
||||
it("plausible interpretations remain in SECTION B, not SECTION A", async () => {
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: eligibleFindings, plausibleInterpretations: plausibleInterps },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
// Should only appear under SECTION B
|
||||
expect(prompt).toContain("Plausible Interpretations:");
|
||||
});
|
||||
|
||||
it("missing situationGraph throws", async () => {
|
||||
await expect(
|
||||
synthesizeInvestigationOverview({}, { provider: fakeProvider() })
|
||||
).rejects.toThrow(/situationGraph is required/);
|
||||
});
|
||||
|
||||
it("non-object situationGraph throws", async () => {
|
||||
await expect(
|
||||
synthesizeInvestigationOverview({ situationGraph: "not an object" }, { provider: fakeProvider() })
|
||||
).rejects.toThrow(/situationGraph is required/);
|
||||
});
|
||||
|
||||
it("findings as non-array throws", async () => {
|
||||
await expect(
|
||||
synthesizeInvestigationOverview({ situationGraph: graph, findings: "string" }, { provider: fakeProvider() })
|
||||
).rejects.toThrow(/findings must be an array/);
|
||||
});
|
||||
|
||||
it("plausibleInterpretations as non-array throws", async () => {
|
||||
await expect(
|
||||
synthesizeInvestigationOverview({ situationGraph: graph, plausibleInterpretations: "string" }, { provider: fakeProvider() })
|
||||
).rejects.toThrow(/plausibleInterpretations must be an array/);
|
||||
});
|
||||
|
||||
it("no provider throws", async () => {
|
||||
await expect(
|
||||
synthesizeInvestigationOverview({ situationGraph: graph, findings: [], plausibleInterpretations: [] }, {})
|
||||
).rejects.toThrow(/OLLAMA_BASE_URL/);
|
||||
});
|
||||
|
||||
it("invalid model response throws validation error", async () => {
|
||||
const fake = fakeProvider(async () => JSON.stringify({ wrongField: "value" }));
|
||||
await expect(
|
||||
synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: [], plausibleInterpretations: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/validation failed/);
|
||||
});
|
||||
|
||||
it("provider throws → propagation", async () => {
|
||||
const fake = fakeProvider(async () => { throw new Error("down"); });
|
||||
await expect(
|
||||
synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: [], plausibleInterpretations: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/provider call failed/);
|
||||
});
|
||||
|
||||
it("graph and findings immutable after synthesis", async () => {
|
||||
const graphSnapshot = JSON.parse(JSON.stringify(graph));
|
||||
const findingsSnapshot = JSON.parse(JSON.stringify(eligibleFindings));
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: eligibleFindings, plausibleInterpretations: plausibleInterps },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(JSON.stringify(graph)).toBe(JSON.stringify(graphSnapshot));
|
||||
expect(JSON.stringify(eligibleFindings)).toBe(JSON.stringify(findingsSnapshot));
|
||||
});
|
||||
|
||||
it("modelName resolved from deps", async () => {
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: [], plausibleInterpretations: [] },
|
||||
{ provider: fake, modelName: "test-model" }
|
||||
);
|
||||
expect(fake.generateReconstruction.mock.calls[0][1]).toBe("test-model");
|
||||
});
|
||||
|
||||
it("empty findings and interpretations produce valid synthesis call", async () => {
|
||||
const fake = fakeProvider();
|
||||
await synthesizeInvestigationOverview(
|
||||
{ situationGraph: graph, findings: [], plausibleInterpretations: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(fake.generateReconstruction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reusable filterEligibleFindings produces correct eligible set for overview", () => {
|
||||
const eligible = filterEligibleFindings(eligibleFindings);
|
||||
expect(eligible.map((f) => f.id)).toEqual(["f1", "f2"]);
|
||||
expect(eligible).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user