517 lines
19 KiB
JavaScript
517 lines
19 KiB
JavaScript
import { describe, expect, it } from "vitest";
|
|
import { makeNode, makeGraph } from "@/lib/graph/schema.js";
|
|
import { formulateQuestionForTarget, buildFocusedDeconstructPrompt, validateFocusedDeconstructSchema } from "@/lib/graph/focused-investigation.js";
|
|
import { FOCUSED_DECONSTRUCT_REGRESSION_CASES, validateRegressionFixtureIntegrity } from "./focused-deconstruct-regression-cases.mjs";
|
|
|
|
// ── helpers ──────────────────────────────────────────────────────────────
|
|
|
|
function makeTestGraph() {
|
|
return makeGraph({
|
|
centralStatement: "Should we launch the product now?",
|
|
currentSummary: "initial summary",
|
|
nodes: [
|
|
makeNode({ id: "a", label: "Market demand signal", description: "Evidence that customers want this.", kind: "unknown", status: "unknown" }),
|
|
makeNode({ id: "b", label: "Competitor activity", description: "What others are doing.", kind: "observation", status: "supported" }),
|
|
],
|
|
edges: [],
|
|
resolvedNodeIds: ["b"],
|
|
});
|
|
}
|
|
|
|
// ── Boundary 1: explicit formulation ─────────────────────────────────────
|
|
|
|
describe("formulateQuestionForTarget — explicit node selection", () => {
|
|
it("validates that the target node exists in the graph", () => {
|
|
const graph = makeTestGraph();
|
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "nonexistent" });
|
|
expect(result.success).toBe(false);
|
|
expect(result.error).toContain("not found");
|
|
});
|
|
|
|
it("validates that the target node is an unknown", () => {
|
|
const graph = makeTestGraph();
|
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "b" });
|
|
expect(result.success).toBe(false);
|
|
expect(result.error).toContain("not an unknown");
|
|
});
|
|
|
|
it("validates that the target node is not resolved", () => {
|
|
const graph = makeTestGraph();
|
|
// Create a resolved unknown node
|
|
graph.nodes.push(makeNode({ id: "c", label: "Done", description: "Completed item", kind: "unknown", status: "resolved" }));
|
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "c" });
|
|
expect(result.success).toBe(false);
|
|
expect(result.error).toContain("resolved");
|
|
});
|
|
|
|
it("uses the supplied targetNodeId in the formulation output", () => {
|
|
const graph = makeTestGraph();
|
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
|
expect(result.success).toBe(true);
|
|
expect(result.targetNodeId).toBe("a");
|
|
});
|
|
|
|
it("does not mutate activeUnknownNodeId on the graph", () => {
|
|
const graph = makeTestGraph();
|
|
const before = graph.activeUnknownNodeId;
|
|
formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
|
expect(graph.activeUnknownNodeId).toBe(before);
|
|
});
|
|
|
|
it("does not mutate SituationGraph in any way", () => {
|
|
const graph = makeTestGraph();
|
|
const snapshot = JSON.stringify(graph);
|
|
formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
|
expect(JSON.stringify(graph)).toBe(snapshot);
|
|
});
|
|
|
|
it("does not invoke global selection (selectActiveUnknownCandidate / determineGraphBackedQuestion)", () => {
|
|
const graph = makeTestGraph();
|
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: "a" });
|
|
expect(result.success).toBe(true);
|
|
expect(result.targetNodeId).toBe("a");
|
|
expect(result.question).toBeDefined();
|
|
expect(typeof result.question).toBe("string");
|
|
expect(result.question.length).toBeGreaterThan(0);
|
|
});
|
|
});
|
|
|
|
// ── Boundary 2: focused answer deconstruction ────────────────────────────
|
|
|
|
describe("buildFocusedDeconstructPrompt", () => {
|
|
it("produces a prompt containing the target label", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "Market demand signal",
|
|
targetDescription: "Evidence that customers want this.",
|
|
centralStatement: "Should we launch?",
|
|
question: "What evidence would clarify market demand?",
|
|
answer: "Some evidence suggests demand.",
|
|
});
|
|
expect(prompt).toContain("Market demand signal");
|
|
});
|
|
|
|
it("produces a prompt containing the answer text", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "Some evidence suggests demand.",
|
|
});
|
|
expect(prompt).toContain("Some evidence suggests demand.");
|
|
});
|
|
|
|
it("does NOT output graph-mutation fields (only lists them as forbidden in the instructions)", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
// The prompt instructs the model NOT to output these fields.
|
|
// It mentions them only as forbidden outputs, not as required outputs.
|
|
expect(prompt).toContain("Do NOT include any of these fields");
|
|
expect(prompt).not.toContain("- addedNodes");
|
|
expect(prompt).not.toContain("- updatedNodes");
|
|
expect(prompt).not.toContain("- removedNodes");
|
|
expect(prompt).not.toContain("- resolvedNodeIds");
|
|
});
|
|
});
|
|
|
|
describe("buildFocusedDeconstructPrompt — relationship rule boundary lock", () => {
|
|
it("relationship instruction explicitly prohibits observation restatement", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Do not create a relationship by merely restating");
|
|
});
|
|
|
|
it("relationship instruction explicitly prohibits promotion of tentative/conditional meaning", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Tentative, speculative, or conditional language must not be promoted into an established relationship");
|
|
});
|
|
|
|
it("relationship instruction requires direct link between two distinct propositions", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("must connect two distinct propositions that the user's answer itself links");
|
|
});
|
|
|
|
it("relationship instruction explicitly permits relationships: [] when no direct link exists", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("relationships: []");
|
|
});
|
|
});
|
|
|
|
describe("validateFocusedDeconstructSchema", () => {
|
|
it("passes when all required fields are present", () => {
|
|
const result = { targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [] };
|
|
expect(validateFocusedDeconstructSchema(result)).toEqual([]);
|
|
});
|
|
|
|
it("fails when a required field is missing", () => {
|
|
const result = { targetNodeId: "a", observations: [], uncertainties: [], assumptions: [] };
|
|
const errors = validateFocusedDeconstructSchema(result);
|
|
expect(errors.length).toBeGreaterThan(0);
|
|
expect(errors.some((e) => e.includes("relationships") || e.includes("possibleFollowUpQuestions"))).toBeTruthy();
|
|
});
|
|
|
|
it("rejects results that contain graph-mutation fields", () => {
|
|
const result = {
|
|
targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [],
|
|
addedNodes: [{ id: "x" }],
|
|
};
|
|
const errors = validateFocusedDeconstructSchema(result);
|
|
expect(errors.some((e) => e.includes("addedNodes"))).toBeTruthy();
|
|
});
|
|
|
|
it("rejects results that contain resolvedNodeIds", () => {
|
|
const result = {
|
|
targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [],
|
|
resolvedNodeIds: ["x"],
|
|
};
|
|
const errors = validateFocusedDeconstructSchema(result);
|
|
expect(errors.some((e) => e.includes("resolvedNodeIds"))).toBeTruthy();
|
|
});
|
|
|
|
it("rejects results that contain activeUnknownNodeId", () => {
|
|
const result = {
|
|
targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [],
|
|
activeUnknownNodeId: "x",
|
|
};
|
|
const errors = validateFocusedDeconstructSchema(result);
|
|
expect(errors.some((e) => e.includes("activeUnknownNodeId"))).toBeTruthy();
|
|
});
|
|
|
|
it("rejects results that contain selectedQuestion", () => {
|
|
const result = {
|
|
targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [],
|
|
selectedQuestion: "some question",
|
|
};
|
|
const errors = validateFocusedDeconstructSchema(result);
|
|
expect(errors.some((e) => e.includes("selectedQuestion"))).toBeTruthy();
|
|
});
|
|
|
|
it("all required fields are observations, uncertainties, assumptions, relationships, possibleFollowUpQuestions", () => {
|
|
const fullResult = { targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [] };
|
|
const errors = validateFocusedDeconstructSchema(fullResult);
|
|
expect(errors.length).toBe(0);
|
|
});
|
|
|
|
it("focused result contains no graph-mutation fields when valid", () => {
|
|
const fullResult = { targetNodeId: "a", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [] };
|
|
const errors = validateFocusedDeconstructSchema(fullResult);
|
|
expect(errors.length).toBe(0);
|
|
// Confirm no forbidden keys are in the result
|
|
const forbidden = ["addedNodes", "updatedNodes", "removedNodes", "addedEdges", "removedEdges", "resolvedNodeIds", "activeUnknownNodeId", "selectedQuestion"];
|
|
for (const key of forbidden) {
|
|
expect(key in fullResult).toBe(false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── Regression fixture integrity (zero live calls) ──────────────────────────
|
|
|
|
describe("FOCUSED_DECONSTRUCT_REGRESSION_CASES fixture integrity", () => {
|
|
it("has exactly 8 cases", () => {
|
|
expect(FOCUSED_DECONSTRUCT_REGRESSION_CASES.length).toBe(8);
|
|
});
|
|
|
|
it("all IDs are unique", () => {
|
|
const ids = FOCUSED_DECONSTRUCT_REGRESSION_CASES.map((c) => c.id);
|
|
expect(new Set(ids).size).toBe(ids.length);
|
|
});
|
|
|
|
it("contains the new structural-frontier case ID", () => {
|
|
const ids = FOCUSED_DECONSTRUCT_REGRESSION_CASES.map((c) => c.id);
|
|
expect(ids).toContain("structural-frontier-over-working-example-detail");
|
|
});
|
|
|
|
it("preserves all original case IDs and includes the new eighth case", () => {
|
|
const ids = FOCUSED_DECONSTRUCT_REGRESSION_CASES.map((c) => c.id);
|
|
for (const origId of [
|
|
"nearest-frontier-wins",
|
|
"no-genuine-assumption",
|
|
"genuine-evidence-sufficiency-assumption",
|
|
"juxtaposition-must-not-create-link",
|
|
"tentative-observation-fidelity",
|
|
"genuine-dependency-frontier",
|
|
"structural-frontier-over-working-example-detail",
|
|
]) {
|
|
expect(ids).toContain(origId);
|
|
}
|
|
expect(ids).toContain("tentative-condition-must-not-map-to-specific-items");
|
|
});
|
|
|
|
it("validateRegressionFixtureIntegrity reports zero errors", () => {
|
|
const errors = validateRegressionFixtureIntegrity();
|
|
expect(errors).toEqual([]);
|
|
});
|
|
});
|
|
|
|
// ── Frontier relevance: prompt contains investigation-relevance anchor ──
|
|
|
|
describe("buildFocusedDeconstructPrompt — frontier is anchored to investigation relevance", () => {
|
|
it("anchors 'nearest' to the central statement and target, not textual adjacency", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain('"nearest" means the smallest investigation-relevant gap');
|
|
expect(prompt).toContain("anchored to the central statement and target description");
|
|
});
|
|
|
|
it("explicitly prevents promoting working-example internal detail over investigation-progressing gaps", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Do NOT promote a detail inside an already-working example");
|
|
});
|
|
|
|
it("requires preferring the gap that advances the current investigation", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("prefer the gap that advances the current investigation");
|
|
});
|
|
|
|
it("introduces working-example / structural-gap disambiguation condition", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("If the answer contains both");
|
|
});
|
|
|
|
it("preserves investigation-relevance qualifier (not just 'smallest unresolved')", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("investigation-relevant gap");
|
|
});
|
|
});
|
|
|
|
// ── Assumption attribution: prompt contains answer-dependence test ──
|
|
|
|
describe("buildFocusedDeconstructPrompt — assumption attribution tightened", () => {
|
|
it("requires an answer-dependence test before attributing assumptions", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Answer-dependence test");
|
|
});
|
|
|
|
it("states the falsity test for attribution", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("could be false and the user's answer would still make complete sense");
|
|
});
|
|
|
|
it("prohibits generalising competence from one observed success", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("One observed success in a single concrete example does NOT by itself establish a general rule");
|
|
});
|
|
|
|
it("explicitly lists prohibited generalisation domains", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("competence, readiness, training, safety, transferability");
|
|
});
|
|
|
|
it("still permits assumptions: [] when no genuine assumption exists", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("assumptions: []");
|
|
});
|
|
|
|
it("does not suppress the original attribution principle", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("what unstated proposition does the user's answer itself rely upon");
|
|
});
|
|
|
|
it("does not suppress the original false-contradiction test", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("would cease to make sense if they were false");
|
|
});
|
|
});
|
|
|
|
// ── Preservation: existing prompt rules still present ──
|
|
|
|
describe("buildFocusedDeconstructPrompt — existing rules preserved", () => {
|
|
it("still requires exactly one uncertainty", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("exactly one string");
|
|
});
|
|
|
|
it("still requires exactly one follow-up question", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("exactly one string — your single best follow-up question");
|
|
});
|
|
|
|
it("still requires plain language (ordinary language for capable persons)", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("ordinary language that a capable person with no specialist vocabulary can understand immediately");
|
|
});
|
|
|
|
it("still prohibits future remedies and branches", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("future constraints");
|
|
});
|
|
|
|
it("still prohibits observation restatement as relationship", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Do not create a relationship by merely restating");
|
|
});
|
|
|
|
it("still prohibits promotion of tentative meaning to relationship", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Tentative, speculative, or conditional language must not be promoted");
|
|
});
|
|
|
|
it("still preserves cross-field ownership rule", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("cross-field ownership");
|
|
});
|
|
|
|
it("still prohibits importing scenario-level inference into assumptions", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Do NOT import plausible interpretations from the wider investigation context");
|
|
});
|
|
|
|
it("still prohibits graph-mutation semantics", () => {
|
|
const prompt = buildFocusedDeconstructPrompt({
|
|
targetLabel: "X",
|
|
targetDescription: "Y",
|
|
centralStatement: "Z",
|
|
question: "Q?",
|
|
answer: "A.",
|
|
});
|
|
expect(prompt).toContain("Do NOT output graph mutations");
|
|
});
|
|
});
|