feat(confidence-engine): isolate finding-informed understanding
This commit is contained in:
@@ -166,3 +166,41 @@ export function applyFindingsToSummary(summary, validatedFindings) {
|
|||||||
|
|
||||||
return text + " [" + parts.join(" | ") + "]";
|
return text + " [" + parts.join(" | ") + "]";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Post-authoritative experimental understanding producer ──
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deterministic proof-seam: produce Current Understanding from validated findings
|
||||||
|
* AFTER authoritative graph update is settled.
|
||||||
|
*
|
||||||
|
* Eligibility contract (closed):
|
||||||
|
* null → eligible provisional working interpretation
|
||||||
|
* "agree" → eligible confirmed evidence
|
||||||
|
* other/falsy → excluded (not_relevant, rejected, etc.)
|
||||||
|
*
|
||||||
|
* Output affects ONLY the summary/current understanding string.
|
||||||
|
*/
|
||||||
|
export function produceFindingInformedSummary(originalSummary, appendedFindings) {
|
||||||
|
if (!appendedFindings || appendedFindings.length === 0) {
|
||||||
|
return originalSummary;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parts = [];
|
||||||
|
|
||||||
|
for (const f of appendedFindings) {
|
||||||
|
if (f.evaluation === "rejected") continue;
|
||||||
|
// null disposition → eligible (provisional working interpretation)
|
||||||
|
// "agree" disposition → eligible (confirmed evidence)
|
||||||
|
if (f.userDisposition === null || f.userDisposition === "agree") {
|
||||||
|
parts.push(f.proposition);
|
||||||
|
}
|
||||||
|
// not_relevant and not_quite → excluded
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parts.length === 0) {
|
||||||
|
return originalSummary; // no eligible findings
|
||||||
|
}
|
||||||
|
|
||||||
|
const evidenceText = `Evidence: [${parts.join("; ")}]`;
|
||||||
|
return originalSummary + " " + evidenceText;
|
||||||
|
}
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import {
|
|||||||
selectActiveUnknownCandidate,
|
selectActiveUnknownCandidate,
|
||||||
validateGraphReferences,
|
validateGraphReferences,
|
||||||
} from "./utils.js";
|
} from "./utils.js";
|
||||||
import { validateFindings } from "./finding-helpers.js";
|
import { validateFindings, produceFindingInformedSummary } from "./finding-helpers.js";
|
||||||
|
|
||||||
function toValidationErrors(error) {
|
function toValidationErrors(error) {
|
||||||
return (
|
return (
|
||||||
@@ -853,6 +853,14 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
|||||||
appendedFindings = normalized;
|
appendedFindings = normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Post-authoritative finding-informed understanding seam ────
|
||||||
|
const authoritativeSummary =
|
||||||
|
applicationResult.updatedSituationGraph?.currentSummary ?? "";
|
||||||
|
const experimentalSummary = produceFindingInformedSummary(
|
||||||
|
authoritativeSummary,
|
||||||
|
appendedFindings,
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
stage: "update_applied",
|
stage: "update_applied",
|
||||||
@@ -866,7 +874,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
|||||||
newActiveUnknownNodeId: applicationResult.newActiveUnknownNodeId,
|
newActiveUnknownNodeId: applicationResult.newActiveUnknownNodeId,
|
||||||
changesApplied: applicationResult.changesApplied,
|
changesApplied: applicationResult.changesApplied,
|
||||||
appendedFindings,
|
appendedFindings,
|
||||||
summary: applicationResult.updatedSituationGraph?.currentSummary ?? "",
|
summary: experimentalSummary,
|
||||||
diagnostics: buildUpdateDiagnostics({
|
diagnostics: buildUpdateDiagnostics({
|
||||||
promptVersion,
|
promptVersion,
|
||||||
modelName,
|
modelName,
|
||||||
|
|||||||
@@ -222,6 +222,34 @@ function makeProposal(overrides = {}) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── v0.49: finding factories for post-authoritative seam test ──
|
||||||
|
|
||||||
|
/** Finding with null disposition → eligible provisional working interpretation */
|
||||||
|
function nullDispositionFinding() {
|
||||||
|
return {
|
||||||
|
id: "finding-a",
|
||||||
|
proposition: "Production volume increased by 12% in Q3.",
|
||||||
|
status: "provisional",
|
||||||
|
userDisposition: null,
|
||||||
|
originatingTargetNodeId: "n-unknown",
|
||||||
|
contributionId: "contrib-test-a",
|
||||||
|
sourceObservation: "user_input",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Finding with not_relevant disposition → excluded from understanding */
|
||||||
|
function notRelevantFinding() {
|
||||||
|
return {
|
||||||
|
id: "finding-b",
|
||||||
|
proposition: "Office humidity levels fluctuate seasonally.",
|
||||||
|
status: "provisional",
|
||||||
|
userDisposition: "not_relevant",
|
||||||
|
originatingTargetNodeId: "n-unknown",
|
||||||
|
contributionId: "contrib-test-b",
|
||||||
|
sourceObservation: "user_input",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function makeComparabilityScenarioGraph() {
|
function makeComparabilityScenarioGraph() {
|
||||||
return makeGraph({
|
return makeGraph({
|
||||||
centralStatement:
|
centralStatement:
|
||||||
@@ -1474,6 +1502,116 @@ describe("lib/graph/orchestrator startCase", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── v0.49: post-authoritative finding-informed understanding seam test ──
|
||||||
|
|
||||||
|
it("post-authoritative findings modify summary but preserve authoritative graph result invariant", async () => {
|
||||||
|
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||||
|
|
||||||
|
const mockGraph = makeGraph({
|
||||||
|
centralStatement: "Complaint counts increased while production also increased.",
|
||||||
|
nodes: [
|
||||||
|
makeNode({
|
||||||
|
id: "n-unknown",
|
||||||
|
label: "Complaint rate denominator",
|
||||||
|
description: "Need the denominator for the complaint rate",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: "n-unknown",
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "Nodes: 1 unknown | Edges: 0 total",
|
||||||
|
});
|
||||||
|
|
||||||
|
const mockApplicationResult = {
|
||||||
|
success: true,
|
||||||
|
updatedSituationGraph: mockGraph,
|
||||||
|
graphUpdate: { addedNodes: [], updatedNodes: [] },
|
||||||
|
selectedQuestion: null,
|
||||||
|
affectedNodeIds: [],
|
||||||
|
resolvedUnknownNodeIds: ["n-unknown"],
|
||||||
|
previousActiveUnknownNodeId: "n-unknown",
|
||||||
|
newActiveUnknownNodeId: null,
|
||||||
|
changesApplied: 1,
|
||||||
|
reasoningState: {
|
||||||
|
comparabilityStatus: "uncertain",
|
||||||
|
comparabilityReason: "deferred",
|
||||||
|
comparabilityEvidence: [],
|
||||||
|
relationshipStatus: "insufficient_information",
|
||||||
|
relationshipReason: "deferred",
|
||||||
|
relationshipAssessed: false,
|
||||||
|
contradictionReasoningAllowed: false,
|
||||||
|
reasoningStages: [],
|
||||||
|
},
|
||||||
|
previousReasoningState: null,
|
||||||
|
graphReferenceValidation: { valid: true, errors: [] },
|
||||||
|
diagnosticEvidence: {},
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyValidatedProposal = vi.fn().mockReturnValue(mockApplicationResult);
|
||||||
|
const provider = {
|
||||||
|
generateReconstruction: vi.fn().mockResolvedValue(makeProposal()),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Control: no findings → summary should remain as authoritative gives
|
||||||
|
const controlRequest = makeUpdateRequest({ situationGraph: mockGraph });
|
||||||
|
const controlResult = await updateCase(controlRequest, {
|
||||||
|
provider,
|
||||||
|
config: MOCK_CONFIG,
|
||||||
|
applyValidatedProposal,
|
||||||
|
applyProposal: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Treatment A: eligible null disposition Finding (schema expects "findings" key)
|
||||||
|
const treatmentAResult = await updateCase(
|
||||||
|
{ ...controlRequest, findings: [nullDispositionFinding()] },
|
||||||
|
{ provider, config: MOCK_CONFIG, applyValidatedProposal, applyProposal: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
// Treatment B: not_relevant Finding (should NOT affect summary)
|
||||||
|
const treatmentBResult = await updateCase(
|
||||||
|
{ ...controlRequest, findings: [notRelevantFinding()] },
|
||||||
|
{ provider, config: MOCK_CONFIG, applyValidatedProposal, applyProposal: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
// 1. Authoritative graph result is structurally identical (control vs treatment)
|
||||||
|
expect(controlResult.success).toBe(true);
|
||||||
|
expect(treatmentAResult.success).toBe(true);
|
||||||
|
expect(treatmentBResult.success).toBe(true);
|
||||||
|
|
||||||
|
expect(controlResult.updatedSituationGraph).toEqual(mockApplicationResult.updatedSituationGraph);
|
||||||
|
expect(treatmentAResult.updatedSituationGraph).toEqual(mockApplicationResult.updatedSituationGraph);
|
||||||
|
expect(treatmentBResult.updatedSituationGraph).toEqual(mockApplicationResult.updatedSituationGraph);
|
||||||
|
|
||||||
|
// 2. summary differs: control == authoritative, treatmentA has evidence appended
|
||||||
|
expect(controlResult.summary).toBe(mockGraph.currentSummary);
|
||||||
|
expect(treatmentAResult.summary).not.toBe(mockGraph.currentSummary);
|
||||||
|
expect(treatmentAResult.summary).toContain("Evidence");
|
||||||
|
expect(treatmentAResult.summary).toContain(
|
||||||
|
"Production volume increased by 12% in Q3.",
|
||||||
|
);
|
||||||
|
|
||||||
|
// 3. not_relevant Finding does NOT affect summary
|
||||||
|
expect(treatmentBResult.summary).toBe(mockGraph.currentSummary);
|
||||||
|
|
||||||
|
// 4. selectedQuestion / frontier fields are identical across all cases
|
||||||
|
expect(controlResult.selectedQuestion).toEqual(treatmentAResult.selectedQuestion);
|
||||||
|
expect(controlResult.selectedQuestion).toEqual(treatmentBResult.selectedQuestion);
|
||||||
|
expect(controlResult.resolvedUnknownNodeIds).toEqual(treatmentAResult.resolvedUnknownNodeIds);
|
||||||
|
expect(controlResult.resolvedUnknownNodeIds).toEqual(treatmentBResult.resolvedUnknownNodeIds);
|
||||||
|
|
||||||
|
// 5. authoritative stages and proposal are identical
|
||||||
|
expect(controlResult.stage).toBe(treatmentAResult.stage);
|
||||||
|
expect(controlResult.stage).toBe(treatmentBResult.stage);
|
||||||
|
expect(controlResult.proposal).toEqual(treatmentAResult.proposal);
|
||||||
|
expect(controlResult.proposal).toEqual(treatmentBResult.proposal);
|
||||||
|
|
||||||
|
// 6. Findings still never enter buildGraphUpdatePrompt (verify provider was called)
|
||||||
|
expect(provider.generateReconstruction).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
|
||||||
it("startCase no longer copies analysis nextQuestion directly when a graph-backed question exists", async () => {
|
it("startCase no longer copies analysis nextQuestion directly when a graph-backed question exists", async () => {
|
||||||
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
|
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
|
||||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||||
|
|||||||
Reference in New Issue
Block a user