fix(focused-investigation): suppress incompatible question strategies
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
const generateReconstruction = vi.fn();
|
||||
|
||||
vi.mock("@/lib/llm/provider", () => ({
|
||||
getProvider: () => ({ generateReconstruction }),
|
||||
getProviderModelName: () => "test-model",
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/supabase/api-auth.js", () => ({
|
||||
withAuthenticatedApi: (handler) => handler,
|
||||
}));
|
||||
|
||||
function makeRecruitmentGraph() {
|
||||
const target = makeNode({
|
||||
id: "recruitment-target",
|
||||
label: "Staffing certainty gap",
|
||||
description:
|
||||
"Whether additional skilled staff needed for expansion can be recruited",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
return makeGraph({
|
||||
centralStatement:
|
||||
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||
currentSummary: "Test summary",
|
||||
nodes: [target],
|
||||
edges: [],
|
||||
activeUnknownNodeId: target.id,
|
||||
resolvedNodeIds: [],
|
||||
});
|
||||
}
|
||||
|
||||
function makeHistoricalGraph() {
|
||||
const target = makeNode({
|
||||
id: "historical-target",
|
||||
label: "Baseline conversion rate",
|
||||
description:
|
||||
"Need the previous baseline because the change cannot be assessed without it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
return makeGraph({
|
||||
centralStatement: "Assess the change against the comparable prior baseline.",
|
||||
currentSummary: "Test summary",
|
||||
nodes: [target],
|
||||
edges: [],
|
||||
activeUnknownNodeId: target.id,
|
||||
resolvedNodeIds: [],
|
||||
});
|
||||
}
|
||||
|
||||
function makeRequest(graph, targetNodeId = graph.nodes[0].id) {
|
||||
return new Request("http://localhost/api/focused-investigation/formulate", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ situationGraph: graph, targetNodeId }),
|
||||
});
|
||||
}
|
||||
|
||||
describe("POST /api/focused-investigation/formulate semantic compatibility", () => {
|
||||
beforeEach(() => {
|
||||
generateReconstruction.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("suppresses an assessor-incompatible strategy without changing the selected target or graph", async () => {
|
||||
const graph = makeRecruitmentGraph();
|
||||
const snapshot = JSON.stringify(graph);
|
||||
generateReconstruction.mockResolvedValue({
|
||||
response: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "incompatible",
|
||||
},
|
||||
});
|
||||
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||
|
||||
const response = await POST(makeRequest(graph));
|
||||
const json = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(json.targetNodeId).toBe("recruitment-target");
|
||||
expect(json.strategy).toBeNull();
|
||||
expect(json.question).toBe(
|
||||
"What evidence would clarify whether additional skilled staff needed for expansion can be recruited?",
|
||||
);
|
||||
expect(JSON.stringify(graph)).toBe(snapshot);
|
||||
expect(generateReconstruction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not let a compatible semantic verdict authorize a grammatically incompatible baseline template", async () => {
|
||||
const graph = makeRecruitmentGraph();
|
||||
generateReconstruction.mockResolvedValue({
|
||||
response: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "compatible",
|
||||
},
|
||||
});
|
||||
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||
|
||||
const response = await POST(makeRequest(graph));
|
||||
const json = await response.json();
|
||||
|
||||
expect(json.strategy).toBeNull();
|
||||
expect(json.question.toLowerCase()).not.toContain("before whether");
|
||||
expect(json.question).toContain("What evidence would clarify whether");
|
||||
});
|
||||
|
||||
it("preserves a compatible historical strategy when its template is suitable", async () => {
|
||||
const graph = makeHistoricalGraph();
|
||||
generateReconstruction.mockResolvedValue({
|
||||
response: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "compatible",
|
||||
},
|
||||
});
|
||||
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||
|
||||
const response = await POST(makeRequest(graph));
|
||||
const json = await response.json();
|
||||
|
||||
expect(json.strategy).toBe("baseline_reconstruction");
|
||||
expect(json.question).toContain("What was the comparable state before");
|
||||
});
|
||||
|
||||
it("treats invalid or mismatched assessment output as non-authorizing", async () => {
|
||||
const graph = makeRecruitmentGraph();
|
||||
generateReconstruction.mockResolvedValue({
|
||||
response: { strategyKey: "evidence_gathering", relevance: "compatible" },
|
||||
});
|
||||
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||
|
||||
const response = await POST(makeRequest(graph));
|
||||
const json = await response.json();
|
||||
|
||||
expect(json.strategy).toBeNull();
|
||||
expect(json.question).toContain("What evidence would clarify whether");
|
||||
});
|
||||
|
||||
it("treats provider unavailability as non-authorizing fallback", async () => {
|
||||
const graph = makeRecruitmentGraph();
|
||||
generateReconstruction.mockRejectedValue(
|
||||
Object.assign(new Error("provider unavailable"), { code: "PROVIDER_UNAVAILABLE" }),
|
||||
);
|
||||
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||
|
||||
const response = await POST(makeRequest(graph));
|
||||
const json = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(json.strategy).toBeNull();
|
||||
expect(json.question).toContain("What evidence would clarify whether");
|
||||
});
|
||||
|
||||
it("aborts the one assessment invocation at the route deadline and falls back", async () => {
|
||||
vi.useFakeTimers();
|
||||
const graph = makeRecruitmentGraph();
|
||||
let observedSignal;
|
||||
generateReconstruction.mockImplementation((_, __, ___, { signal }) => {
|
||||
observedSignal = signal;
|
||||
return new Promise((_, reject) => {
|
||||
signal.addEventListener("abort", () => {
|
||||
const error = new Error("assessment aborted");
|
||||
error.name = "AbortError";
|
||||
reject(error);
|
||||
}, { once: true });
|
||||
});
|
||||
});
|
||||
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||
|
||||
const responsePromise = POST(makeRequest(graph));
|
||||
await vi.advanceTimersByTimeAsync(10000);
|
||||
const response = await responsePromise;
|
||||
const json = await response.json();
|
||||
|
||||
expect(observedSignal.aborted).toBe(true);
|
||||
expect(json.strategy).toBeNull();
|
||||
expect(json.question).toContain("What evidence would clarify whether");
|
||||
});
|
||||
});
|
||||
@@ -65,6 +65,40 @@ describe("formulateQuestionForTarget — explicit node selection", () => {
|
||||
expect(JSON.stringify(graph)).toBe(snapshot);
|
||||
});
|
||||
|
||||
it("forwards an ephemeral incompatible compatibility verdict without changing the selected target or graph", () => {
|
||||
const unknown = makeNode({
|
||||
id: "historical-target",
|
||||
label: "Earlier comparable state",
|
||||
description: "Whether the prior staffing level was sufficient for expansion",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"Compare the prior staffing baseline with the level needed before the expansion.",
|
||||
currentSummary: "initial summary",
|
||||
nodes: [unknown],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
});
|
||||
const snapshot = JSON.stringify(graph);
|
||||
|
||||
const result = formulateQuestionForTarget({
|
||||
situationGraph: graph,
|
||||
targetNodeId: unknown.id,
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "incompatible",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.targetNodeId).toBe(unknown.id);
|
||||
expect(result.strategy).toBeNull();
|
||||
expect(JSON.stringify(graph)).toBe(snapshot);
|
||||
});
|
||||
|
||||
it("does not invoke global selection (selectActiveUnknownCandidate / determineGraphBackedQuestion)", () => {
|
||||
const graph = makeTestGraph();
|
||||
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
||||
|
||||
@@ -1614,6 +1614,252 @@ describe("60B.75 — real decision detection for sufficiency question", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── Experiment 3 Iteration 3 — Deterministic Reproduction Recovery ──────────
|
||||
// Hypothesis: A Whether ... uncertainty passed into baseline_reconstruction
|
||||
// interpolates into a template requiring a noun phrase, producing malformed output.
|
||||
// Fixed input: "Whether additional skilled staff needed for expansion can be recruited"
|
||||
|
||||
describe("Experiment 3 — Whether proposition in baseline reconstruction", () => {
|
||||
it("suppresses the reproduced malformed baseline template at the focused-investigation compatibility boundary", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-experiment3-staffing",
|
||||
label: "Staffing certainty gap",
|
||||
description:
|
||||
"Whether additional skilled staff needed for expansion can be recruited",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||
nodes: [unknown],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
|
||||
const proposed = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
// Preserve the deterministic reproduction that identified the bug.
|
||||
expect(proposed.strategy).toBe("baseline_reconstruction");
|
||||
expect(proposed.question).toBe(
|
||||
"What was the comparable state before whether additional skilled staff needed for expansion can be recruited?",
|
||||
);
|
||||
|
||||
const corrected = formulateQuestion({
|
||||
node: unknown,
|
||||
graph,
|
||||
context: {
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "compatible",
|
||||
},
|
||||
strategyTemplateCompatibility: "incompatible",
|
||||
},
|
||||
});
|
||||
|
||||
expect(corrected.strategy).toBeNull();
|
||||
expect(corrected.question).toBe(
|
||||
"What evidence would clarify whether additional skilled staff needed for expansion can be recruited?",
|
||||
);
|
||||
expect(corrected.question.toLowerCase()).not.toContain("before whether");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Experiment 14 — Deterministic Compatibility Verdict Contract ───────────
|
||||
|
||||
describe("Experiment 14 — strategy compatibility verdict", () => {
|
||||
function makeBaselineWhetherFixture() {
|
||||
const unknown = makeNode({
|
||||
id: "n-experiment14-staffing",
|
||||
label: "Staffing certainty gap",
|
||||
description:
|
||||
"Whether additional skilled staff needed for expansion can be recruited",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||
});
|
||||
return { unknown, graph };
|
||||
}
|
||||
|
||||
it("preserves existing strategy and question when no verdict is supplied", () => {
|
||||
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.strategy).toBe("baseline_reconstruction");
|
||||
expect(result.question).toBe(
|
||||
"What was the comparable state before whether additional skilled staff needed for expansion can be recruited?",
|
||||
);
|
||||
});
|
||||
|
||||
it("permits the matching proposed strategy when the verdict is compatible", () => {
|
||||
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph,
|
||||
context: {
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "compatible",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("baseline_reconstruction");
|
||||
expect(result.question).toBe(
|
||||
"What was the comparable state before whether additional skilled staff needed for expansion can be recruited?",
|
||||
);
|
||||
});
|
||||
|
||||
it("suppresses an incompatible proposed strategy and uses the existing evidence path", () => {
|
||||
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||
const graphSnapshot = JSON.stringify(graph);
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph,
|
||||
context: {
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "incompatible",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.strategy).toBeNull();
|
||||
expect(result.question).toBe(
|
||||
"What evidence would clarify whether additional skilled staff needed for expansion can be recruited?",
|
||||
);
|
||||
expect(JSON.stringify(graph)).toBe(graphSnapshot);
|
||||
});
|
||||
|
||||
it("does not let invalid or mismatched verdicts authorise a proposed strategy", () => {
|
||||
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||
const invalid = formulateQuestion({
|
||||
node: unknown,
|
||||
graph,
|
||||
context: {
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "unknown",
|
||||
},
|
||||
},
|
||||
});
|
||||
const mismatched = formulateQuestion({
|
||||
node: unknown,
|
||||
graph,
|
||||
context: {
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "evidence_gathering",
|
||||
relevance: "compatible",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(invalid.strategy).toBeNull();
|
||||
expect(mismatched.strategy).toBeNull();
|
||||
expect(invalid.question).toContain("What evidence would clarify whether");
|
||||
expect(mismatched.question).toContain("What evidence would clarify whether");
|
||||
});
|
||||
|
||||
it("allows a compatible historical strategy for a coherent Whether-proposition", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-experiment14-historical",
|
||||
label: "Earlier comparable state",
|
||||
description: "Whether the prior staffing level was sufficient for expansion",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"Compare the prior staffing baseline with the level needed before the expansion.",
|
||||
});
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph,
|
||||
context: {
|
||||
strategyCompatibilityVerdict: {
|
||||
strategyKey: "baseline_reconstruction",
|
||||
relevance: "compatible",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("baseline_reconstruction");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Experiment 6 — Graph-Context Causal Comparison ─────────────────────────────
|
||||
// Hypothesis: baseline_reconstruction strategy on the recruitment unknown is driven
|
||||
// by historical-baseline vocabulary in graph-level context (centralStatement), not
|
||||
// in the recruitment uncertainty itself. Removing that vocabulary should prevent
|
||||
// the specific baseline-selection route while preserving all node-level input.
|
||||
|
||||
describe("Experiment 6 — Isolate graph-context influence on strategy selection", () => {
|
||||
it("removing baseline vocabulary from centralStatement changes strategy away from baseline_reconstruction", () => {
|
||||
// ── shared open-question fixture (identical in both variants) ──
|
||||
const openQuestion = makeNode({
|
||||
id: "n-experiment3-staffing",
|
||||
label: "Staffing certainty gap",
|
||||
description:
|
||||
"Whether additional skilled staff needed for expansion can be recruited",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
// ── Variant A — baseline context present (existing fixture) ──
|
||||
const graphA = makeGraph({
|
||||
centralStatement:
|
||||
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||
nodes: [openQuestion],
|
||||
edges: [],
|
||||
activeUnknownNodeId: openQuestion.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
|
||||
// ── Variant B — baseline context absent (neutral wording) ──
|
||||
const graphB = makeGraph({
|
||||
centralStatement:
|
||||
"Current staffing levels are being reviewed as we prepare for the planned expansion.",
|
||||
nodes: [openQuestion],
|
||||
edges: [],
|
||||
activeUnknownNodeId: openQuestion.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
|
||||
// ── Execute both variants ──
|
||||
const resultA = formulateQuestion({ node: openQuestion, graph: graphA });
|
||||
const resultB = formulateQuestion({ node: openQuestion, graph: graphB });
|
||||
|
||||
// Record for report
|
||||
console.log("[Exp6-A] strategy:", resultA.strategy, "| question:", resultA.question);
|
||||
console.log("[Exp6-B] strategy:", resultB.strategy, "| question:", resultB.question);
|
||||
|
||||
// Causal contrast: baseline_reconstruction requires BOTH graph-level AND node-level
|
||||
// baseline vocabulary to match the first (strict) selection path.
|
||||
// Variant A must select baseline_reconstruction.
|
||||
expect(resultA.strategy).toBe("baseline_reconstruction");
|
||||
|
||||
// Variant B must NOT select baseline_reconstruction (graph-level baseline removed).
|
||||
expect(resultB.strategy).not.toBe("baseline_reconstruction");
|
||||
|
||||
// The malformed "before whether" output must disappear in Variant B.
|
||||
expect(resultB.question.toLowerCase()).not.toContain(
|
||||
"before whether",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 60B.80 — sufficiency re-selection after proposal application (no false State B) ────
|
||||
|
||||
describe("60B.80 — no false State B after proposal adds unresolved material factor", () => {
|
||||
|
||||
Reference in New Issue
Block a user