fix(focused-investigation): suppress incompatible question strategies
This commit is contained in:
@@ -1,6 +1,13 @@
|
|||||||
import { formulateQuestionForTarget } from "@/lib/graph/focused-investigation";
|
import { getProvider, getProviderModelName } from "@/lib/llm/provider";
|
||||||
|
import {
|
||||||
|
assessStrategyCompatibility,
|
||||||
|
formulateQuestionForTarget,
|
||||||
|
isStrategyQuestionTemplateCompatible,
|
||||||
|
} from "@/lib/graph/focused-investigation";
|
||||||
import { withAuthenticatedApi } from "@/lib/supabase/api-auth.js";
|
import { withAuthenticatedApi } from "@/lib/supabase/api-auth.js";
|
||||||
|
|
||||||
|
const STRATEGY_ASSESSMENT_TIMEOUT_MS = 10000;
|
||||||
|
|
||||||
async function post(request) {
|
async function post(request) {
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
@@ -19,18 +26,59 @@ async function post(request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = formulateQuestionForTarget({
|
const proposed = formulateQuestionForTarget({
|
||||||
situationGraph: body.situationGraph,
|
situationGraph: body.situationGraph,
|
||||||
targetNodeId: body.targetNodeId,
|
targetNodeId: body.targetNodeId,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!result.success) {
|
if (!proposed.success) {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ success: false, error: result.error },
|
{ success: false, error: proposed.error },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let result = proposed;
|
||||||
|
if (proposed.investigationStrategy) {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeout = setTimeout(
|
||||||
|
() => controller.abort(),
|
||||||
|
STRATEGY_ASSESSMENT_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
let strategyCompatibilityVerdict = {
|
||||||
|
strategyKey: proposed.investigationStrategy.key,
|
||||||
|
relevance: "unavailable",
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const assessment = await assessStrategyCompatibility({
|
||||||
|
provider: getProvider(),
|
||||||
|
modelName: getProviderModelName(),
|
||||||
|
targetLabel: body.situationGraph.nodes.find((node) => node.id === body.targetNodeId)?.label,
|
||||||
|
targetDescription: body.situationGraph.nodes.find((node) => node.id === body.targetNodeId)?.description,
|
||||||
|
centralStatement: body.situationGraph.centralStatement,
|
||||||
|
proposedStrategy: proposed.investigationStrategy,
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
if (assessment.success) strategyCompatibilityVerdict = assessment.verdict;
|
||||||
|
} catch {
|
||||||
|
// A timed-out or unavailable assessment is explicitly non-authorizing.
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
result = formulateQuestionForTarget({
|
||||||
|
situationGraph: body.situationGraph,
|
||||||
|
targetNodeId: body.targetNodeId,
|
||||||
|
strategyCompatibilityVerdict,
|
||||||
|
strategyTemplateCompatibility:
|
||||||
|
strategyCompatibilityVerdict.relevance === "compatible" &&
|
||||||
|
!isStrategyQuestionTemplateCompatible(proposed.investigationStrategy)
|
||||||
|
? "incompatible"
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return Response.json({
|
return Response.json({
|
||||||
success: true,
|
success: true,
|
||||||
targetNodeId: result.targetNodeId,
|
targetNodeId: result.targetNodeId,
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
|
import {
|
||||||
|
formulateQuestion,
|
||||||
|
isStrategyQuestionTemplateCompatible,
|
||||||
|
} from "@/lib/graph/question-formulator.js";
|
||||||
|
|
||||||
const FOCUSED_ANSWER_SCHEMA_FIELDS = [
|
const FOCUSED_ANSWER_SCHEMA_FIELDS = [
|
||||||
"targetNodeId",
|
"targetNodeId",
|
||||||
@@ -35,6 +38,16 @@ export const focusedDeconstructJsonSchema = {
|
|||||||
additionalProperties: false,
|
additionalProperties: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const strategyCompatibilityAssessmentJsonSchema = {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
strategyKey: { type: "string" },
|
||||||
|
relevance: { type: "string", enum: ["compatible", "incompatible"] },
|
||||||
|
},
|
||||||
|
required: ["strategyKey", "relevance"],
|
||||||
|
additionalProperties: false,
|
||||||
|
};
|
||||||
|
|
||||||
const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
|
const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
|
||||||
"addedNodes",
|
"addedNodes",
|
||||||
"updatedNodes",
|
"updatedNodes",
|
||||||
@@ -49,14 +62,19 @@ const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
|
|||||||
/**
|
/**
|
||||||
* Formulate a question for the explicitly user-selected unresolved node.
|
* Formulate a question for the explicitly user-selected unresolved node.
|
||||||
*
|
*
|
||||||
* Input: { situationGraph, targetNodeId }
|
* Input: { situationGraph, targetNodeId, strategyCompatibilityVerdict? }
|
||||||
* Validates that target exists, is an unknown, and is not resolved.
|
* Validates that target exists, is an unknown, and is not resolved.
|
||||||
* Calls formulateQuestion with only factual explicit-user-targeting context.
|
* Calls formulateQuestion with only factual explicit-user-targeting context.
|
||||||
* Does NOT invoke selectActiveUnknownCandidate, determineGraphBackedQuestion,
|
* Does NOT invoke selectActiveUnknownCandidate, determineGraphBackedQuestion,
|
||||||
* global ranking, or global recommendation.
|
* global ranking, or global recommendation.
|
||||||
* Does NOT mutate activeUnknownNodeId, selectedQuestion, SituationGraph, or resolvedNodeIds.
|
* Does NOT mutate activeUnknownNodeId, selectedQuestion, SituationGraph, or resolvedNodeIds.
|
||||||
*/
|
*/
|
||||||
export function formulateQuestionForTarget({ situationGraph, targetNodeId }) {
|
export function formulateQuestionForTarget({
|
||||||
|
situationGraph,
|
||||||
|
targetNodeId,
|
||||||
|
strategyCompatibilityVerdict,
|
||||||
|
strategyTemplateCompatibility,
|
||||||
|
}) {
|
||||||
const nodesById = new Map(situationGraph?.nodes?.map((n) => [n.id, n]) || []);
|
const nodesById = new Map(situationGraph?.nodes?.map((n) => [n.id, n]) || []);
|
||||||
const targetNode = nodesById.get(targetNodeId);
|
const targetNode = nodesById.get(targetNodeId);
|
||||||
|
|
||||||
@@ -80,6 +98,12 @@ export function formulateQuestionForTarget({ situationGraph, targetNodeId }) {
|
|||||||
_explicitTargetNodeId: targetNodeId,
|
_explicitTargetNodeId: targetNodeId,
|
||||||
_explicitTargetLabel: targetNode.label,
|
_explicitTargetLabel: targetNode.label,
|
||||||
_explicitTargetDescription: targetNode.description,
|
_explicitTargetDescription: targetNode.description,
|
||||||
|
...(strategyCompatibilityVerdict === undefined
|
||||||
|
? {}
|
||||||
|
: { strategyCompatibilityVerdict }),
|
||||||
|
...(strategyTemplateCompatibility === undefined
|
||||||
|
? {}
|
||||||
|
: { strategyTemplateCompatibility }),
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = formulateQuestion({ node: targetNode, graph: situationGraph, context });
|
const result = formulateQuestion({ node: targetNode, graph: situationGraph, context });
|
||||||
@@ -89,6 +113,7 @@ export function formulateQuestionForTarget({ situationGraph, targetNodeId }) {
|
|||||||
targetNodeId,
|
targetNodeId,
|
||||||
question: result.question,
|
question: result.question,
|
||||||
strategy: result.strategy ?? null,
|
strategy: result.strategy ?? null,
|
||||||
|
investigationStrategy: result.investigationStrategy,
|
||||||
reasoningPattern: result.reasoningPattern,
|
reasoningPattern: result.reasoningPattern,
|
||||||
reasoningPatternReason: result.reasoningPatternReason,
|
reasoningPatternReason: result.reasoningPatternReason,
|
||||||
reason: result.reason,
|
reason: result.reason,
|
||||||
@@ -99,6 +124,84 @@ export function formulateQuestionForTarget({ situationGraph, targetNodeId }) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function buildStrategyCompatibilityAssessmentPrompt({
|
||||||
|
targetLabel,
|
||||||
|
targetDescription,
|
||||||
|
centralStatement,
|
||||||
|
proposedStrategy,
|
||||||
|
}) {
|
||||||
|
return `Assess whether one already-proposed investigation strategy is relevant to one explicitly selected uncertainty.
|
||||||
|
|
||||||
|
Return exactly one JSON object and JSON only.
|
||||||
|
|
||||||
|
You may only assess the relevance of the supplied proposed strategy to the supplied selected uncertainty.
|
||||||
|
Do not choose another uncertainty. Do not propose another strategy. Do not write the final question. Do not decide the user's conclusion. Do not mutate or recommend changes to any graph.
|
||||||
|
|
||||||
|
Return "compatible" only when this exact strategy would help investigate this exact uncertainty. Return "incompatible" when it would not.
|
||||||
|
|
||||||
|
Selected uncertainty:
|
||||||
|
- label: ${targetLabel}
|
||||||
|
- description: ${targetDescription}
|
||||||
|
|
||||||
|
Case context:
|
||||||
|
- central statement: ${centralStatement}
|
||||||
|
|
||||||
|
Proposed strategy:
|
||||||
|
- strategyKey: ${proposedStrategy.key}
|
||||||
|
- strategy meaning: ${proposedStrategy.meaning}
|
||||||
|
|
||||||
|
Return exactly:
|
||||||
|
{"strategyKey":"${proposedStrategy.key}","relevance":"compatible" | "incompatible"}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateStrategyCompatibilityVerdict(result, proposedStrategy) {
|
||||||
|
if (!result || typeof result !== "object" || Array.isArray(result)) return null;
|
||||||
|
const keys = Object.keys(result);
|
||||||
|
if (
|
||||||
|
keys.length !== 2 ||
|
||||||
|
!keys.includes("strategyKey") ||
|
||||||
|
!keys.includes("relevance") ||
|
||||||
|
typeof result.strategyKey !== "string" ||
|
||||||
|
result.strategyKey !== proposedStrategy?.key ||
|
||||||
|
(result.relevance !== "compatible" && result.relevance !== "incompatible")
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { strategyKey: result.strategyKey, relevance: result.relevance };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function assessStrategyCompatibility({
|
||||||
|
provider,
|
||||||
|
modelName,
|
||||||
|
targetLabel,
|
||||||
|
targetDescription,
|
||||||
|
centralStatement,
|
||||||
|
proposedStrategy,
|
||||||
|
signal,
|
||||||
|
}) {
|
||||||
|
const prompt = buildStrategyCompatibilityAssessmentPrompt({
|
||||||
|
targetLabel,
|
||||||
|
targetDescription,
|
||||||
|
centralStatement,
|
||||||
|
proposedStrategy,
|
||||||
|
});
|
||||||
|
const wrapper = await provider.generateReconstruction(
|
||||||
|
prompt,
|
||||||
|
modelName,
|
||||||
|
strategyCompatibilityAssessmentJsonSchema,
|
||||||
|
{ signal },
|
||||||
|
);
|
||||||
|
const verdict = validateStrategyCompatibilityVerdict(
|
||||||
|
wrapper?.response,
|
||||||
|
proposedStrategy,
|
||||||
|
);
|
||||||
|
return verdict
|
||||||
|
? { success: true, verdict }
|
||||||
|
: { success: false, status: "invalid" };
|
||||||
|
}
|
||||||
|
|
||||||
|
export { isStrategyQuestionTemplateCompatible };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build a focused answer deconstruction prompt for one explicitly user-chosen investigation.
|
* Build a focused answer deconstruction prompt for one explicitly user-chosen investigation.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -1198,6 +1198,37 @@ function constrainStrategyToReasoningPattern(strategy, reasoningPattern) {
|
|||||||
return allowedStrategies.includes(strategy.key) ? strategy : null;
|
return allowedStrategies.includes(strategy.key) ? strategy : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function applyStrategyCompatibilityVerdict(proposedStrategy, verdict) {
|
||||||
|
// No verdict is the backward-compatible path. A supplied verdict can only
|
||||||
|
// preserve this exact proposal or suppress it; it cannot select a substitute.
|
||||||
|
if (verdict === undefined) return proposedStrategy;
|
||||||
|
if (!proposedStrategy || !verdict || typeof verdict !== "object" || Array.isArray(verdict)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const keys = Object.keys(verdict);
|
||||||
|
if (
|
||||||
|
keys.length !== 2 ||
|
||||||
|
!keys.includes("strategyKey") ||
|
||||||
|
!keys.includes("relevance") ||
|
||||||
|
typeof verdict.strategyKey !== "string" ||
|
||||||
|
(verdict.relevance !== "compatible" && verdict.relevance !== "incompatible") ||
|
||||||
|
verdict.strategyKey !== proposedStrategy.key
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return verdict.relevance === "compatible" ? proposedStrategy : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isStrategyQuestionTemplateCompatible(strategy) {
|
||||||
|
if (!strategy) return true;
|
||||||
|
return !(
|
||||||
|
strategy.key === "baseline_reconstruction" &&
|
||||||
|
isWhetherPropositionMeaning(strategy.meaning)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function selectQuestionFamily({
|
function selectQuestionFamily({
|
||||||
node,
|
node,
|
||||||
graph,
|
graph,
|
||||||
@@ -2047,7 +2078,7 @@ export function formulateQuestion({ node, graph, context = {} }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const investigationStrategy = selectInvestigationStrategy({
|
const proposedInvestigationStrategy = selectInvestigationStrategy({
|
||||||
node,
|
node,
|
||||||
graph,
|
graph,
|
||||||
context: {
|
context: {
|
||||||
@@ -2055,12 +2086,20 @@ export function formulateQuestion({ node, graph, context = {} }) {
|
|||||||
reasoningPattern: reasoningPatternSelection.pattern,
|
reasoningPattern: reasoningPatternSelection.pattern,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const investigationStrategy = applyStrategyCompatibilityVerdict(
|
||||||
|
proposedInvestigationStrategy,
|
||||||
|
context.strategyCompatibilityVerdict,
|
||||||
|
);
|
||||||
|
const templateCompatibleStrategy =
|
||||||
|
context.strategyTemplateCompatibility === "incompatible"
|
||||||
|
? null
|
||||||
|
: investigationStrategy;
|
||||||
|
|
||||||
const questionFamilySelection = selectQuestionFamily({
|
const questionFamilySelection = selectQuestionFamily({
|
||||||
node,
|
node,
|
||||||
graph,
|
graph,
|
||||||
reasoningPattern: reasoningPatternSelection.pattern,
|
reasoningPattern: reasoningPatternSelection.pattern,
|
||||||
investigationStrategy,
|
investigationStrategy: templateCompatibleStrategy,
|
||||||
});
|
});
|
||||||
|
|
||||||
let question = buildQuestionFromFamily({
|
let question = buildQuestionFromFamily({
|
||||||
@@ -2069,7 +2108,7 @@ export function formulateQuestion({ node, graph, context = {} }) {
|
|||||||
reasoningPattern: reasoningPatternSelection.pattern,
|
reasoningPattern: reasoningPatternSelection.pattern,
|
||||||
questionFamily: questionFamilySelection.family,
|
questionFamily: questionFamilySelection.family,
|
||||||
selectedQuestionTemplate: questionFamilySelection.template,
|
selectedQuestionTemplate: questionFamilySelection.template,
|
||||||
investigationStrategy,
|
investigationStrategy: templateCompatibleStrategy,
|
||||||
});
|
});
|
||||||
|
|
||||||
question = sanitizeQuestionText(question);
|
question = sanitizeQuestionText(question);
|
||||||
@@ -2080,11 +2119,11 @@ export function formulateQuestion({ node, graph, context = {} }) {
|
|||||||
if (
|
if (
|
||||||
!validateFormulatedQuestion(
|
!validateFormulatedQuestion(
|
||||||
question,
|
question,
|
||||||
investigationStrategy?.meaning || fallbackMeaning,
|
templateCompatibleStrategy?.meaning || fallbackMeaning,
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
question = sanitizeQuestionText(
|
question = sanitizeQuestionText(
|
||||||
(investigationStrategy &&
|
(templateCompatibleStrategy &&
|
||||||
isClaimLikeUnknown(
|
isClaimLikeUnknown(
|
||||||
node,
|
node,
|
||||||
normaliseText(
|
normaliseText(
|
||||||
@@ -2112,11 +2151,11 @@ export function formulateQuestion({ node, graph, context = {} }) {
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
question,
|
question,
|
||||||
reason: investigationStrategy
|
reason: templateCompatibleStrategy
|
||||||
? `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`
|
? `Formulated from graph context using the ${templateCompatibleStrategy.key} investigation strategy.`
|
||||||
: "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.",
|
: "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.",
|
||||||
strategy: investigationStrategy?.key ?? null,
|
strategy: templateCompatibleStrategy?.key ?? null,
|
||||||
investigationStrategy,
|
investigationStrategy: templateCompatibleStrategy,
|
||||||
reasoningPattern: reasoningPatternSelection.pattern,
|
reasoningPattern: reasoningPatternSelection.pattern,
|
||||||
reasoningPatternReason: reasoningPatternSelection.reason,
|
reasoningPatternReason: reasoningPatternSelection.reason,
|
||||||
questionFamily: questionFamilySelection.family,
|
questionFamily: questionFamilySelection.family,
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||||
|
|
||||||
|
const generateReconstruction = vi.fn();
|
||||||
|
|
||||||
|
vi.mock("@/lib/llm/provider", () => ({
|
||||||
|
getProvider: () => ({ generateReconstruction }),
|
||||||
|
getProviderModelName: () => "test-model",
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("@/lib/supabase/api-auth.js", () => ({
|
||||||
|
withAuthenticatedApi: (handler) => handler,
|
||||||
|
}));
|
||||||
|
|
||||||
|
function makeRecruitmentGraph() {
|
||||||
|
const target = makeNode({
|
||||||
|
id: "recruitment-target",
|
||||||
|
label: "Staffing certainty gap",
|
||||||
|
description:
|
||||||
|
"Whether additional skilled staff needed for expansion can be recruited",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
return makeGraph({
|
||||||
|
centralStatement:
|
||||||
|
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||||
|
currentSummary: "Test summary",
|
||||||
|
nodes: [target],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: target.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeHistoricalGraph() {
|
||||||
|
const target = makeNode({
|
||||||
|
id: "historical-target",
|
||||||
|
label: "Baseline conversion rate",
|
||||||
|
description:
|
||||||
|
"Need the previous baseline because the change cannot be assessed without it.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
return makeGraph({
|
||||||
|
centralStatement: "Assess the change against the comparable prior baseline.",
|
||||||
|
currentSummary: "Test summary",
|
||||||
|
nodes: [target],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: target.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeRequest(graph, targetNodeId = graph.nodes[0].id) {
|
||||||
|
return new Request("http://localhost/api/focused-investigation/formulate", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ situationGraph: graph, targetNodeId }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("POST /api/focused-investigation/formulate semantic compatibility", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
generateReconstruction.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("suppresses an assessor-incompatible strategy without changing the selected target or graph", async () => {
|
||||||
|
const graph = makeRecruitmentGraph();
|
||||||
|
const snapshot = JSON.stringify(graph);
|
||||||
|
generateReconstruction.mockResolvedValue({
|
||||||
|
response: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "incompatible",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||||
|
|
||||||
|
const response = await POST(makeRequest(graph));
|
||||||
|
const json = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(json.targetNodeId).toBe("recruitment-target");
|
||||||
|
expect(json.strategy).toBeNull();
|
||||||
|
expect(json.question).toBe(
|
||||||
|
"What evidence would clarify whether additional skilled staff needed for expansion can be recruited?",
|
||||||
|
);
|
||||||
|
expect(JSON.stringify(graph)).toBe(snapshot);
|
||||||
|
expect(generateReconstruction).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not let a compatible semantic verdict authorize a grammatically incompatible baseline template", async () => {
|
||||||
|
const graph = makeRecruitmentGraph();
|
||||||
|
generateReconstruction.mockResolvedValue({
|
||||||
|
response: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "compatible",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||||
|
|
||||||
|
const response = await POST(makeRequest(graph));
|
||||||
|
const json = await response.json();
|
||||||
|
|
||||||
|
expect(json.strategy).toBeNull();
|
||||||
|
expect(json.question.toLowerCase()).not.toContain("before whether");
|
||||||
|
expect(json.question).toContain("What evidence would clarify whether");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves a compatible historical strategy when its template is suitable", async () => {
|
||||||
|
const graph = makeHistoricalGraph();
|
||||||
|
generateReconstruction.mockResolvedValue({
|
||||||
|
response: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "compatible",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||||
|
|
||||||
|
const response = await POST(makeRequest(graph));
|
||||||
|
const json = await response.json();
|
||||||
|
|
||||||
|
expect(json.strategy).toBe("baseline_reconstruction");
|
||||||
|
expect(json.question).toContain("What was the comparable state before");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats invalid or mismatched assessment output as non-authorizing", async () => {
|
||||||
|
const graph = makeRecruitmentGraph();
|
||||||
|
generateReconstruction.mockResolvedValue({
|
||||||
|
response: { strategyKey: "evidence_gathering", relevance: "compatible" },
|
||||||
|
});
|
||||||
|
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||||
|
|
||||||
|
const response = await POST(makeRequest(graph));
|
||||||
|
const json = await response.json();
|
||||||
|
|
||||||
|
expect(json.strategy).toBeNull();
|
||||||
|
expect(json.question).toContain("What evidence would clarify whether");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats provider unavailability as non-authorizing fallback", async () => {
|
||||||
|
const graph = makeRecruitmentGraph();
|
||||||
|
generateReconstruction.mockRejectedValue(
|
||||||
|
Object.assign(new Error("provider unavailable"), { code: "PROVIDER_UNAVAILABLE" }),
|
||||||
|
);
|
||||||
|
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||||
|
|
||||||
|
const response = await POST(makeRequest(graph));
|
||||||
|
const json = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(json.strategy).toBeNull();
|
||||||
|
expect(json.question).toContain("What evidence would clarify whether");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("aborts the one assessment invocation at the route deadline and falls back", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const graph = makeRecruitmentGraph();
|
||||||
|
let observedSignal;
|
||||||
|
generateReconstruction.mockImplementation((_, __, ___, { signal }) => {
|
||||||
|
observedSignal = signal;
|
||||||
|
return new Promise((_, reject) => {
|
||||||
|
signal.addEventListener("abort", () => {
|
||||||
|
const error = new Error("assessment aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
reject(error);
|
||||||
|
}, { once: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const { POST } = await import("@/app/api/focused-investigation/formulate/route.js");
|
||||||
|
|
||||||
|
const responsePromise = POST(makeRequest(graph));
|
||||||
|
await vi.advanceTimersByTimeAsync(10000);
|
||||||
|
const response = await responsePromise;
|
||||||
|
const json = await response.json();
|
||||||
|
|
||||||
|
expect(observedSignal.aborted).toBe(true);
|
||||||
|
expect(json.strategy).toBeNull();
|
||||||
|
expect(json.question).toContain("What evidence would clarify whether");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -65,6 +65,40 @@ describe("formulateQuestionForTarget — explicit node selection", () => {
|
|||||||
expect(JSON.stringify(graph)).toBe(snapshot);
|
expect(JSON.stringify(graph)).toBe(snapshot);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards an ephemeral incompatible compatibility verdict without changing the selected target or graph", () => {
|
||||||
|
const unknown = makeNode({
|
||||||
|
id: "historical-target",
|
||||||
|
label: "Earlier comparable state",
|
||||||
|
description: "Whether the prior staffing level was sufficient for expansion",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
});
|
||||||
|
const graph = makeGraph({
|
||||||
|
centralStatement:
|
||||||
|
"Compare the prior staffing baseline with the level needed before the expansion.",
|
||||||
|
currentSummary: "initial summary",
|
||||||
|
nodes: [unknown],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: unknown.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
});
|
||||||
|
const snapshot = JSON.stringify(graph);
|
||||||
|
|
||||||
|
const result = formulateQuestionForTarget({
|
||||||
|
situationGraph: graph,
|
||||||
|
targetNodeId: unknown.id,
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "incompatible",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.targetNodeId).toBe(unknown.id);
|
||||||
|
expect(result.strategy).toBeNull();
|
||||||
|
expect(JSON.stringify(graph)).toBe(snapshot);
|
||||||
|
});
|
||||||
|
|
||||||
it("does not invoke global selection (selectActiveUnknownCandidate / determineGraphBackedQuestion)", () => {
|
it("does not invoke global selection (selectActiveUnknownCandidate / determineGraphBackedQuestion)", () => {
|
||||||
const graph = makeTestGraph();
|
const graph = makeTestGraph();
|
||||||
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
||||||
|
|||||||
@@ -1614,6 +1614,252 @@ describe("60B.75 — real decision detection for sufficiency question", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── Experiment 3 Iteration 3 — Deterministic Reproduction Recovery ──────────
|
||||||
|
// Hypothesis: A Whether ... uncertainty passed into baseline_reconstruction
|
||||||
|
// interpolates into a template requiring a noun phrase, producing malformed output.
|
||||||
|
// Fixed input: "Whether additional skilled staff needed for expansion can be recruited"
|
||||||
|
|
||||||
|
describe("Experiment 3 — Whether proposition in baseline reconstruction", () => {
|
||||||
|
it("suppresses the reproduced malformed baseline template at the focused-investigation compatibility boundary", () => {
|
||||||
|
const unknown = makeNode({
|
||||||
|
id: "n-experiment3-staffing",
|
||||||
|
label: "Staffing certainty gap",
|
||||||
|
description:
|
||||||
|
"Whether additional skilled staff needed for expansion can be recruited",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
|
||||||
|
const graph = makeGraph({
|
||||||
|
centralStatement:
|
||||||
|
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||||
|
nodes: [unknown],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: unknown.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "Test summary",
|
||||||
|
});
|
||||||
|
|
||||||
|
const proposed = formulateQuestion({ node: unknown, graph });
|
||||||
|
|
||||||
|
// Preserve the deterministic reproduction that identified the bug.
|
||||||
|
expect(proposed.strategy).toBe("baseline_reconstruction");
|
||||||
|
expect(proposed.question).toBe(
|
||||||
|
"What was the comparable state before whether additional skilled staff needed for expansion can be recruited?",
|
||||||
|
);
|
||||||
|
|
||||||
|
const corrected = formulateQuestion({
|
||||||
|
node: unknown,
|
||||||
|
graph,
|
||||||
|
context: {
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "compatible",
|
||||||
|
},
|
||||||
|
strategyTemplateCompatibility: "incompatible",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(corrected.strategy).toBeNull();
|
||||||
|
expect(corrected.question).toBe(
|
||||||
|
"What evidence would clarify whether additional skilled staff needed for expansion can be recruited?",
|
||||||
|
);
|
||||||
|
expect(corrected.question.toLowerCase()).not.toContain("before whether");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Experiment 14 — Deterministic Compatibility Verdict Contract ───────────
|
||||||
|
|
||||||
|
describe("Experiment 14 — strategy compatibility verdict", () => {
|
||||||
|
function makeBaselineWhetherFixture() {
|
||||||
|
const unknown = makeNode({
|
||||||
|
id: "n-experiment14-staffing",
|
||||||
|
label: "Staffing certainty gap",
|
||||||
|
description:
|
||||||
|
"Whether additional skilled staff needed for expansion can be recruited",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
const graph = makeGraphFor(unknown, {
|
||||||
|
centralStatement:
|
||||||
|
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||||
|
});
|
||||||
|
return { unknown, graph };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("preserves existing strategy and question when no verdict is supplied", () => {
|
||||||
|
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||||
|
const result = formulateQuestion({ node: unknown, graph });
|
||||||
|
|
||||||
|
expect(result.strategy).toBe("baseline_reconstruction");
|
||||||
|
expect(result.question).toBe(
|
||||||
|
"What was the comparable state before whether additional skilled staff needed for expansion can be recruited?",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("permits the matching proposed strategy when the verdict is compatible", () => {
|
||||||
|
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||||
|
const result = formulateQuestion({
|
||||||
|
node: unknown,
|
||||||
|
graph,
|
||||||
|
context: {
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "compatible",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.strategy).toBe("baseline_reconstruction");
|
||||||
|
expect(result.question).toBe(
|
||||||
|
"What was the comparable state before whether additional skilled staff needed for expansion can be recruited?",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("suppresses an incompatible proposed strategy and uses the existing evidence path", () => {
|
||||||
|
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||||
|
const graphSnapshot = JSON.stringify(graph);
|
||||||
|
const result = formulateQuestion({
|
||||||
|
node: unknown,
|
||||||
|
graph,
|
||||||
|
context: {
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "incompatible",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.strategy).toBeNull();
|
||||||
|
expect(result.question).toBe(
|
||||||
|
"What evidence would clarify whether additional skilled staff needed for expansion can be recruited?",
|
||||||
|
);
|
||||||
|
expect(JSON.stringify(graph)).toBe(graphSnapshot);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not let invalid or mismatched verdicts authorise a proposed strategy", () => {
|
||||||
|
const { unknown, graph } = makeBaselineWhetherFixture();
|
||||||
|
const invalid = formulateQuestion({
|
||||||
|
node: unknown,
|
||||||
|
graph,
|
||||||
|
context: {
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "unknown",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const mismatched = formulateQuestion({
|
||||||
|
node: unknown,
|
||||||
|
graph,
|
||||||
|
context: {
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "evidence_gathering",
|
||||||
|
relevance: "compatible",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(invalid.strategy).toBeNull();
|
||||||
|
expect(mismatched.strategy).toBeNull();
|
||||||
|
expect(invalid.question).toContain("What evidence would clarify whether");
|
||||||
|
expect(mismatched.question).toContain("What evidence would clarify whether");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("allows a compatible historical strategy for a coherent Whether-proposition", () => {
|
||||||
|
const unknown = makeNode({
|
||||||
|
id: "n-experiment14-historical",
|
||||||
|
label: "Earlier comparable state",
|
||||||
|
description: "Whether the prior staffing level was sufficient for expansion",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
const graph = makeGraphFor(unknown, {
|
||||||
|
centralStatement:
|
||||||
|
"Compare the prior staffing baseline with the level needed before the expansion.",
|
||||||
|
});
|
||||||
|
const result = formulateQuestion({
|
||||||
|
node: unknown,
|
||||||
|
graph,
|
||||||
|
context: {
|
||||||
|
strategyCompatibilityVerdict: {
|
||||||
|
strategyKey: "baseline_reconstruction",
|
||||||
|
relevance: "compatible",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.strategy).toBe("baseline_reconstruction");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Experiment 6 — Graph-Context Causal Comparison ─────────────────────────────
|
||||||
|
// Hypothesis: baseline_reconstruction strategy on the recruitment unknown is driven
|
||||||
|
// by historical-baseline vocabulary in graph-level context (centralStatement), not
|
||||||
|
// in the recruitment uncertainty itself. Removing that vocabulary should prevent
|
||||||
|
// the specific baseline-selection route while preserving all node-level input.
|
||||||
|
|
||||||
|
describe("Experiment 6 — Isolate graph-context influence on strategy selection", () => {
|
||||||
|
it("removing baseline vocabulary from centralStatement changes strategy away from baseline_reconstruction", () => {
|
||||||
|
// ── shared open-question fixture (identical in both variants) ──
|
||||||
|
const openQuestion = makeNode({
|
||||||
|
id: "n-experiment3-staffing",
|
||||||
|
label: "Staffing certainty gap",
|
||||||
|
description:
|
||||||
|
"Whether additional skilled staff needed for expansion can be recruited",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Variant A — baseline context present (existing fixture) ──
|
||||||
|
const graphA = makeGraph({
|
||||||
|
centralStatement:
|
||||||
|
"We need to compare our current baseline against the comparable state before staffing is secured for expansion.",
|
||||||
|
nodes: [openQuestion],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: openQuestion.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "Test summary",
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Variant B — baseline context absent (neutral wording) ──
|
||||||
|
const graphB = makeGraph({
|
||||||
|
centralStatement:
|
||||||
|
"Current staffing levels are being reviewed as we prepare for the planned expansion.",
|
||||||
|
nodes: [openQuestion],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: openQuestion.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "Test summary",
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Execute both variants ──
|
||||||
|
const resultA = formulateQuestion({ node: openQuestion, graph: graphA });
|
||||||
|
const resultB = formulateQuestion({ node: openQuestion, graph: graphB });
|
||||||
|
|
||||||
|
// Record for report
|
||||||
|
console.log("[Exp6-A] strategy:", resultA.strategy, "| question:", resultA.question);
|
||||||
|
console.log("[Exp6-B] strategy:", resultB.strategy, "| question:", resultB.question);
|
||||||
|
|
||||||
|
// Causal contrast: baseline_reconstruction requires BOTH graph-level AND node-level
|
||||||
|
// baseline vocabulary to match the first (strict) selection path.
|
||||||
|
// Variant A must select baseline_reconstruction.
|
||||||
|
expect(resultA.strategy).toBe("baseline_reconstruction");
|
||||||
|
|
||||||
|
// Variant B must NOT select baseline_reconstruction (graph-level baseline removed).
|
||||||
|
expect(resultB.strategy).not.toBe("baseline_reconstruction");
|
||||||
|
|
||||||
|
// The malformed "before whether" output must disappear in Variant B.
|
||||||
|
expect(resultB.question.toLowerCase()).not.toContain(
|
||||||
|
"before whether",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// ── 60B.80 — sufficiency re-selection after proposal application (no false State B) ────
|
// ── 60B.80 — sufficiency re-selection after proposal application (no false State B) ────
|
||||||
|
|
||||||
describe("60B.80 — no false State B after proposal adds unresolved material factor", () => {
|
describe("60B.80 — no false State B after proposal adds unresolved material factor", () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user