284 lines
11 KiB
JavaScript
284 lines
11 KiB
JavaScript
import { describe, expect, it } from "vitest";
|
|
import { buildGraphUpdatePrompt } from "@/lib/graph/prompt-builder.js";
|
|
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
|
|
|
function makeContext() {
|
|
const unknown = makeNode({
|
|
id: "n-unknown",
|
|
label: "Complaint rate denominator",
|
|
description: "Need the denominator to compare complaint rates",
|
|
kind: "unknown",
|
|
status: "unknown",
|
|
confidence: "high",
|
|
});
|
|
const observation = makeNode({
|
|
id: "n-obs",
|
|
label: "Complaints up 35%",
|
|
description: "Complaints increased by 35%",
|
|
kind: "observation",
|
|
status: "supported",
|
|
confidence: "high",
|
|
});
|
|
|
|
return {
|
|
situationGraph: makeGraph({
|
|
centralStatement: "Complaints increased while production increased.",
|
|
nodes: [unknown, observation],
|
|
edges: [
|
|
makeEdge({
|
|
id: "e1",
|
|
fromNodeId: observation.id,
|
|
toNodeId: unknown.id,
|
|
relationship: "supports",
|
|
confidence: "high",
|
|
description: "Observation informs the unknown",
|
|
}),
|
|
],
|
|
activeUnknownNodeId: unknown.id,
|
|
resolvedNodeIds: [],
|
|
currentSummary:
|
|
"Nodes: 1 observation, 1 unknown | Edges: 1 total | Unknowns: 1 unresolved",
|
|
}),
|
|
previousQuestion: "What denominator is being used for the complaint rate?",
|
|
answer:
|
|
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
|
};
|
|
}
|
|
|
|
describe("buildGraphUpdatePrompt", () => {
|
|
it("includes the current graph", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain(
|
|
"Complaints increased while production increased.",
|
|
);
|
|
expect(prompt).toContain("Complaint rate denominator");
|
|
});
|
|
|
|
it("includes previous question and answer", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain(
|
|
"What denominator is being used for the complaint rate?",
|
|
);
|
|
expect(prompt).toContain(
|
|
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
|
);
|
|
});
|
|
|
|
it("contains exact schema keys", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("addedNodes");
|
|
expect(prompt).toContain("updatedNodes");
|
|
expect(prompt).toContain("addedEdges");
|
|
expect(prompt).toContain("removedEdgeIds");
|
|
expect(prompt).toContain("resolvedUnknownNodeIds");
|
|
expect(prompt).toContain("affectedNodeIds");
|
|
expect(prompt).toContain("selectedQuestion");
|
|
expect(prompt).toContain("answerMeaning");
|
|
});
|
|
|
|
it("lists enum values", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain(
|
|
"observation | reported_claim | metric | state | transition | relationship | assumption | unknown | conclusion",
|
|
);
|
|
expect(prompt).toContain(
|
|
"known | unknown | provisional | supported | weakened | contradicted | resolved",
|
|
);
|
|
expect(prompt).toContain(
|
|
"supports | weakens | contradicts | depends_on | causes | may_cause | measures | compares_with | updates | other",
|
|
);
|
|
});
|
|
|
|
it("forbids full-graph replacement", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("Never return a replacement graph");
|
|
expect(prompt).toContain("Propose changes only");
|
|
});
|
|
|
|
it("requires JSON only", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("Return JSON only");
|
|
expect(prompt).toContain("Return one JSON object only");
|
|
});
|
|
|
|
it("describes controlled emergent unknown rules", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("Add at most 3 new unknown nodes");
|
|
expect(prompt).toContain("Resolve the answered unknown first");
|
|
expect(prompt).toContain(
|
|
"selectedQuestion.question must be one narrow non-compound question",
|
|
);
|
|
expect(prompt).toContain(
|
|
"the engine will deterministically choose final priority after validation",
|
|
);
|
|
});
|
|
|
|
it("instructs the model to preserve user-supported meaning separately from inference", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain(
|
|
"answerMeaning.userSupportedMeaning must state only what the user's answer directly supports",
|
|
);
|
|
expect(prompt).toContain(
|
|
"Put any stronger interpretation in answerMeaning.possibleInference",
|
|
);
|
|
expect(prompt).toContain(
|
|
"supportCategory and resolutionGuidance are optional descriptive hints only",
|
|
);
|
|
expect(prompt).toContain(
|
|
"leave them null rather than inventing rigid category labels",
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Semantic-to-mutation contract (57J.39) ──────────────
|
|
|
|
describe("buildGraphUpdatePrompt — semantic-to-mutation MUST rule", () => {
|
|
it("contains the explicit structural-materialization MUST rule", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("MUST express its effect through structural mutation");
|
|
});
|
|
|
|
it("rule permits update/refine of existing structure", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("update/refinement of existing structure");
|
|
});
|
|
|
|
it("rule permits resolving an existing unknown", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("resolution of an existing unknown");
|
|
});
|
|
|
|
it("rule permits genuinely new unknown when needed", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("a genuinely new unknown");
|
|
});
|
|
|
|
it("rule explicitly states answerMeaning alone is not sufficient", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
expect(prompt).toContain("answerMeaning alone is not sufficient");
|
|
});
|
|
|
|
it("rule does NOT force adding a new node", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
// The rule should be silent about forcing new nodes — this is preserved by existing rule #7.
|
|
// Verify the MUST rule exists but doesn't contain "must add a new node" or similar.
|
|
const mustRuleMatch = prompt.match(
|
|
/6\..*?(?=\n7\.)/s,
|
|
);
|
|
expect(mustRuleMatch).not.toBe(null);
|
|
expect(mustRuleMatch[0]).not.toContain("must add a new node");
|
|
});
|
|
|
|
it("does not imply possibleInference alone triggers mutation", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
// The rule must reference userSupportedMeaning specifically, not possibleInference as a trigger.
|
|
const mustRuleMatch = prompt.match(
|
|
/6\..*?(?=\n7\.)/s,
|
|
);
|
|
expect(mustRuleMatch[0]).toContain("userSupportedMeaning");
|
|
});
|
|
|
|
// ── 57J.43 — no surviving semantic-only/no-op conflict ──
|
|
|
|
it("PASS: no direct contradiction — MUST rule is not undermined by empty-array permission", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
|
|
// The MUST rule must exist...
|
|
expect(prompt).toContain(
|
|
"MUST express its effect through structural mutation",
|
|
);
|
|
|
|
// ...and the empty-array permission must NOT be unconditional.
|
|
// It must reference rule #6 as a condition, meaning it cannot apply
|
|
// when the MUST rule fires.
|
|
const additionalGuidance = prompt.split("## Additional Guidance")[1];
|
|
|
|
// The old conflicting wording must be absent:
|
|
expect(additionalGuidance).not.toContain(
|
|
"If the answer does not justify a change, return empty arrays",
|
|
);
|
|
|
|
// The new permission must reference rule #6:
|
|
expect(additionalGuidance).toContain("rule #6");
|
|
});
|
|
|
|
it("PASS: legitimate true no-op preserved — empty mutation allowed when rule #6 does not apply", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
const additionalGuidance = prompt.split("## Additional Guidance")[1];
|
|
|
|
// The corrected bullet must still allow empty arrays, but only
|
|
// when rule #6 does not apply (no consequential meaning).
|
|
expect(additionalGuidance).toContain(
|
|
"return empty arrays for every category",
|
|
);
|
|
// And it must be conditioned:
|
|
expect(additionalGuidance).toContain("does not apply");
|
|
});
|
|
|
|
it("PASS: answerMeaning is not structural progress", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
const additionalGuidance = prompt.split("## Additional Guidance")[1];
|
|
|
|
// Must explicitly separate answerMeaning from graph mutation:
|
|
expect(additionalGuidance).toContain("preserves semantic fidelity");
|
|
// And must not say answerMeaning alone can substitute for mutation:
|
|
expect(additionalGuidance).not.toContain(
|
|
"even when the graph change remains unresolved",
|
|
);
|
|
});
|
|
|
|
it("PASS: duplicate protection preserved — no weakening of existing-duplicate rules", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
// Rule #4 and rule #11 must still exist with their substance:
|
|
expect(prompt).toContain("genuinely new concepts");
|
|
expect(prompt).toContain("duplicate unknowns");
|
|
// Additional Guidance preference for update over add:
|
|
const additionalGuidance = prompt.split("## Additional Guidance")[1];
|
|
expect(additionalGuidance).toContain("prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes");
|
|
});
|
|
|
|
it("PASS: update/refine route preserved — no new mandatory-add requirement", () => {
|
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
|
|
|
// The MUST rule permits update/refine (not just add):
|
|
expect(prompt).toContain("update/refinement of existing structure");
|
|
// Additional guidance still encourages preferring updates:
|
|
const additionalGuidance = prompt.split("## Additional Guidance")[1];
|
|
expect(additionalGuidance).toContain("prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes");
|
|
expect(additionalGuidance).toContain("update that node rather than creating only a parallel observation");
|
|
});
|
|
|
|
it("PASS: possibleInference separation preserved — not converted to mandatory mutation", () => {
|
|
const fullPrompt = buildGraphUpdatePrompt(makeContext());
|
|
|
|
// Rule #27 must still separate possibleInference from userSupportedMeaning:
|
|
expect(fullPrompt).toContain(
|
|
"Put any stronger interpretation in answerMeaning.possibleInference, not in userSupportedMeaning",
|
|
);
|
|
|
|
// The corrected Additional Guidance must reference the mutation trigger via rule #6
|
|
// (which itself references userSupportedMeaning), not possibleInference:
|
|
const additionalGuidance = fullPrompt.split("## Additional Guidance")[1];
|
|
expect(additionalGuidance).toContain("rule #6");
|
|
expect(additionalGuidance).not.toContain("possibleInference");
|
|
|
|
// Rule #27 exists in the prompt (separation preserved):
|
|
expect(fullPrompt).toContain("answerMeaning.possibleInference, not in userSupportedMeaning");
|
|
});
|
|
|
|
it("PASS: no action-selection machinery added — no keyword routing or node-kind decision table", () => {
|
|
const fullPrompt = buildGraphUpdatePrompt(makeContext());
|
|
|
|
// Confirm we did not add new provider-specific routing:
|
|
expect(fullPrompt).not.toContain("qwen");
|
|
expect(fullPrompt).not.toContain("claude");
|
|
expect(fullPrompt).not.toContain("gpt");
|
|
|
|
// No keyword-based node-kind decision table:
|
|
expect(fullPrompt).not.toContain("keyword");
|
|
|
|
// No mandatory-add logic:
|
|
expect(fullPrompt).not.toContain("must add a new node");
|
|
});
|
|
});
|