import { describe, expect, it } from "vitest"; import { buildGraphUpdatePrompt } from "@/lib/graph/prompt-builder.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; function makeContext() { const unknown = makeNode({ id: "n-unknown", label: "Complaint rate denominator", description: "Need the denominator to compare complaint rates", kind: "unknown", status: "unknown", confidence: "high", }); const observation = makeNode({ id: "n-obs", label: "Complaints up 35%", description: "Complaints increased by 35%", kind: "observation", status: "supported", confidence: "high", }); return { situationGraph: makeGraph({ centralStatement: "Complaints increased while production increased.", nodes: [unknown, observation], edges: [ makeEdge({ id: "e1", fromNodeId: observation.id, toNodeId: unknown.id, relationship: "supports", confidence: "high", description: "Observation informs the unknown", }), ], activeUnknownNodeId: unknown.id, resolvedNodeIds: [], currentSummary: "Nodes: 1 observation, 1 unknown | Edges: 1 total | Unknowns: 1 unresolved", }), previousQuestion: "What denominator is being used for the complaint rate?", answer: "The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.", }; } describe("buildGraphUpdatePrompt", () => { it("includes the current graph", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "Complaints increased while production increased.", ); expect(prompt).toContain("Complaint rate denominator"); }); it("includes previous question and answer", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "What denominator is being used for the complaint rate?", ); expect(prompt).toContain( "The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.", ); }); it("contains exact schema keys", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("addedNodes"); expect(prompt).toContain("updatedNodes"); expect(prompt).toContain("addedEdges"); expect(prompt).toContain("removedEdgeIds"); expect(prompt).toContain("resolvedUnknownNodeIds"); expect(prompt).toContain("affectedNodeIds"); expect(prompt).toContain("selectedQuestion"); expect(prompt).toContain("answerMeaning"); }); it("lists enum values", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "observation | reported_claim | metric | state | transition | relationship | assumption | unknown | conclusion", ); expect(prompt).toContain( "known | unknown | provisional | supported | weakened | contradicted | resolved", ); expect(prompt).toContain( "supports | weakens | contradicts | depends_on | causes | may_cause | measures | compares_with | updates | other", ); }); it("forbids full-graph replacement", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("Never return a replacement graph"); expect(prompt).toContain("Propose changes only"); }); it("requires JSON only", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("Return JSON only"); expect(prompt).toContain("Return one JSON object only"); }); it("describes controlled emergent unknown rules", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("Add at most 3 new unknown nodes"); expect(prompt).toContain("Resolve the answered unknown first"); expect(prompt).toContain( "selectedQuestion.question must be one narrow non-compound question", ); expect(prompt).toContain( "the engine will deterministically choose final priority after validation", ); }); it("instructs the model to preserve user-supported meaning separately from inference", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "answerMeaning.userSupportedMeaning must state only what the user's answer directly supports", ); expect(prompt).toContain( "Put any stronger interpretation in answerMeaning.possibleInference", ); expect(prompt).toContain( "populate supportCategory with one of the allowed values whenever the user's meaning fits an existing category", ); expect(prompt).toContain( "Do not leave supportCategory null merely because the wording is uncertain", ); }); it("lists allowed structured semantic enum values", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("## Allowed answerMeaning.supportCategory Values"); expect(prompt).toContain( "relative_priority_only | conditional_tradeoff | uncertain | explicit_hard_constraint | other", ); expect(prompt).toContain( "must_remain_unresolved | may_resolve | must_resolve", ); }); it("strengthens resolutionGuidance population guidance without provider-specific wording", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "Populate resolutionGuidance when the user's meaning genuinely implies must_remain_unresolved, may_resolve, or must_resolve", ); expect(prompt).toContain( "Keep it null only when no existing resolution state actually applies", ); expect(prompt).not.toContain("qwen"); }); }); // ── Semantic-to-mutation contract (57J.39) ────────────── describe("buildGraphUpdatePrompt — semantic-to-mutation MUST rule", () => { it("contains the explicit structural-materialization MUST rule", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "MUST express its effect through structural mutation", ); }); it("rule permits update/refine of existing structure", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("update/refinement of existing structure"); }); it("rule permits resolving an existing unknown", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("resolution of an existing unknown"); }); it("rule permits genuinely new unknown when needed", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("a genuinely new unknown"); }); it("rule explicitly states answerMeaning alone is not sufficient", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("answerMeaning alone is not sufficient"); }); it("rule does NOT force adding a new node", () => { const prompt = buildGraphUpdatePrompt(makeContext()); // The rule should be silent about forcing new nodes — this is preserved by existing rule #7. // Verify the MUST rule exists but doesn't contain "must add a new node" or similar. const mustRuleMatch = prompt.match(/6\..*?(?=\n7\.)/s); expect(mustRuleMatch).not.toBe(null); expect(mustRuleMatch[0]).not.toContain("must add a new node"); }); it("does not imply possibleInference alone triggers mutation", () => { const prompt = buildGraphUpdatePrompt(makeContext()); // The rule must reference userSupportedMeaning specifically, not possibleInference as a trigger. const mustRuleMatch = prompt.match(/6\..*?(?=\n7\.)/s); expect(mustRuleMatch[0]).toContain("userSupportedMeaning"); }); // ── 57J.43 — no surviving semantic-only/no-op conflict ── it("PASS: no direct contradiction — MUST rule is not undermined by empty-array permission", () => { const prompt = buildGraphUpdatePrompt(makeContext()); // The MUST rule must exist... expect(prompt).toContain( "MUST express its effect through structural mutation", ); // ...and the empty-array permission must NOT be unconditional. // It must reference rule #6 as a condition, meaning it cannot apply // when the MUST rule fires. const additionalGuidance = prompt.split("## Additional Guidance")[1]; // The old conflicting wording must be absent: expect(additionalGuidance).not.toContain( "If the answer does not justify a change, return empty arrays", ); // The new permission must reference rule #6: expect(additionalGuidance).toContain("rule #6"); }); it("PASS: legitimate true no-op preserved — empty mutation allowed when rule #6 does not apply", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const additionalGuidance = prompt.split("## Additional Guidance")[1]; // The corrected bullet must still allow empty arrays, but only // when rule #6 does not apply (no consequential meaning). expect(additionalGuidance).toContain( "return empty arrays for every category", ); // And it must be conditioned: expect(additionalGuidance).toContain("does not apply"); }); it("PASS: answerMeaning is not structural progress", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const additionalGuidance = prompt.split("## Additional Guidance")[1]; // Must explicitly separate answerMeaning from graph mutation: expect(additionalGuidance).toContain("preserves semantic fidelity"); // And must not say answerMeaning alone can substitute for mutation: expect(additionalGuidance).not.toContain( "even when the graph change remains unresolved", ); }); it("PASS: duplicate protection preserved — no weakening of existing-duplicate rules", () => { const prompt = buildGraphUpdatePrompt(makeContext()); // Rule #4 and rule #11 must still exist with their substance: expect(prompt).toContain("genuinely new concepts"); expect(prompt).toContain("duplicate unknowns"); // Additional Guidance preference for update over add: const additionalGuidance = prompt.split("## Additional Guidance")[1]; expect(additionalGuidance).toContain( "prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes", ); }); it("PASS: update/refine route preserved — no new mandatory-add requirement", () => { const prompt = buildGraphUpdatePrompt(makeContext()); // The MUST rule permits update/refine (not just add): expect(prompt).toContain("update/refinement of existing structure"); // Additional guidance still encourages preferring updates: const additionalGuidance = prompt.split("## Additional Guidance")[1]; expect(additionalGuidance).toContain( "prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes", ); expect(additionalGuidance).toContain( "update that node rather than creating only a parallel observation", ); }); it("PASS: possibleInference separation preserved — not converted to mandatory mutation", () => { const fullPrompt = buildGraphUpdatePrompt(makeContext()); // Rule #27 must still separate possibleInference from userSupportedMeaning: expect(fullPrompt).toContain( "Put any stronger interpretation in answerMeaning.possibleInference, not in userSupportedMeaning", ); // The corrected Additional Guidance must reference the mutation trigger via rule #6 // (which itself references userSupportedMeaning), not possibleInference: const additionalGuidance = fullPrompt.split("## Additional Guidance")[1]; expect(additionalGuidance).toContain("rule #6"); expect(additionalGuidance).not.toContain("possibleInference"); // Rule #27 exists in the prompt (separation preserved): expect(fullPrompt).toContain( "answerMeaning.possibleInference, not in userSupportedMeaning", ); }); it("PASS: no action-selection machinery added — no keyword routing or node-kind decision table", () => { const fullPrompt = buildGraphUpdatePrompt(makeContext()); // Confirm we did not add new provider-specific routing: expect(fullPrompt).not.toContain("qwen"); expect(fullPrompt).not.toContain("claude"); expect(fullPrompt).not.toContain("gpt"); // No keyword-based node-kind decision table: expect(fullPrompt).not.toContain("keyword"); // No mandatory-add logic: expect(fullPrompt).not.toContain("must add a new node"); }); }); // ── Experiment 57J.46 — existing-first uncertainty fallback ─── describe("buildGraphUpdatePrompt — 57J.46 existing-first uncertainty fallback", () => { function getAdditionalGuidance(prompt) { return prompt.split("## Additional Guidance")[1]; } // Test 1 — existing-first ordering exists it("test 1: assembled prompt explicitly says to check for an equivalent unresolved unknown first", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain( "first check whether an existing unresolved node", ); expect(guidance).toContain("represents the same uncertainty"); }); // Test 2 — reuse path explicit it("test 2: prompt says update/refine existing structure rather than add a duplicate when equivalent exists", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("if so, update/refine that existing structure"); expect(guidance).toContain("rather than adding a duplicate"); }); // Test 3 — fallback-to-add explicit it("test 3: prompt explicitly requires adding a new unknown when no equivalent unresolved unknown exists", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("if no such node exists"); expect(guidance).toContain( "add a new unknown that directly represents the unresolved uncertainty", ); }); // Test 4 — ordered fallback means: existing first, otherwise add it("test 4: full ordered fallback is present in the assembled prompt", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); // The entire rule must be present as a single coherent instruction: expect(guidance).toContain( "When rule #6 applies to explicitly unresolved uncertainty: first check whether an existing unresolved node already represents the same uncertainty; if so, update/refine that existing structure rather than adding a duplicate; if no such node exists, add a new unknown that directly represents the unresolved uncertainty; do not use an edge alone to represent a previously unrepresented uncertainty.", ); }); // Test 5 — merely related node is insufficient it("test 5: prompt does not imply a general related state/cost node counts as representing the same uncertainty", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); // The rule requires "same uncertainty" — not "related" or "similar": expect(guidance).toContain("same uncertainty"); // Must not use weaker criteria: expect(guidance).not.toContain("similar"); expect(guidance).not.toContain("related to"); }); // Test 6 — edge-only insufficient it("test 6: prompt explicitly prevents using an edge alone to represent previously unrepresented uncertainty", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain( "do not use an edge alone to represent a previously unrepresented uncertainty", ); }); // Test 7 — possibleInference does not create unknowns it("test 7: existing possibleInference separation remains intact — rule #27 still separates interpretation from userSupportedMeaning", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "Put any stronger interpretation in answerMeaning.possibleInference, not in userSupportedMeaning", ); // The 57J.46 rule must NOT reference possibleInference as a trigger: const guidance = getAdditionalGuidance(prompt); expect(guidance).not.toContain("possibleInference"); }); // Test 8 — actual resolution path preserved it("test 8: prompt still allows resolution when the user's answer genuinely resolves an existing unknown", () => { const prompt = buildGraphUpdatePrompt(makeContext()); // The "resolve existing" guidance must remain intact: expect(prompt).toContain( "include that existing node ID in resolvedUnknownNodeIds", ); expect(prompt).toContain( "update that node rather than creating only a parallel observation", ); // Rule #5 (resolve answered unknown first) must still exist: expect(prompt).toContain( "Resolve the answered unknown first when the answer supports it", ); }); // Test 9 — duplicate validator/contract preserved it("test 9: existing duplicate-avoidance wording remains unchanged", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("Do not add duplicate unknowns"); expect(prompt).toContain("genuinely new concepts"); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain( "prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes", ); }); // Test 10 — scope remains uncertainty-only (not universal to all categories) it("test 10: the new action-order rule does not apply universally to facts, constraints, decisions, or other meaning categories", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); // The rule must be scoped to "unresolved uncertainty" specifically: expect(guidance).toContain("explicitly unresolved uncertainty"); // It must not say "any answer" or "all categories": expect(guidance).not.toContain("any answer"); expect(guidance).not.toContain("every change"); expect(guidance).not.toContain("all categories"); // Rule #7 (new unknown conditions) and other category rules must be untouched: expect(prompt).toContain( "Add new unknown nodes only when the answer introduces a new decision, claim, object, measure, dependency, or unresolved term directly relevant to the case", ); }); // Test 11 — fidelity / possibleInference separation preserved it("test 11: fidelity rule separating userSupportedMeaning from possibleInference is untouched", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "answerMeaning.userSupportedMeaning must state only what the user's answer directly supports", ); }); // Test 12 — traceability preserved it("test 12: traceability requirement for new unknowns remains intact", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain( "Every new unknown must be directly traceable to the user's answer and its description must state why that uncertainty matters", ); }); // Test 13 — no noop validator changed it("test 13: the 'rule #6 does not apply → empty arrays' permission is preserved unchanged", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain( "If rule #6 does not apply (the answer contains no user-supported meaning that requires graph progress) and there is no other justification for change, return empty arrays for every category", ); }); // Test 14 — no semantic classifier or keyword logic added it("test 14: no new deterministic semantic matcher or keyword matching was added", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).not.toContain("threshold"); expect(prompt).not.toContain("synonym"); expect(prompt).not.toContain("keyword match"); }); // Test 15 — provider-agnostic preserved it("test 15: no provider-specific wording added", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).not.toContain("qwen"); expect(prompt).not.toContain("claude"); expect(prompt).not.toContain("gpt"); expect(prompt).not.toContain("ollama"); }); }); // ── Experiment 57J.55 — uncertainty identity vs topical overlap ───────── describe("buildGraphUpdatePrompt — 57J.55 uncertainty identity vs topical overlap", () => { function getAdditionalGuidance(prompt) { return prompt.split("## Additional Guidance")[1]; } // Test 1 — "same uncertainty" defined by same resolution question it("test 1: 'same uncertainty' is defined in terms of the same resolution question", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("same resolution question"); }); // Test 2 — topical overlap explicitly insufficient it("test 2: topical overlap is explicitly not sufficient for 'same uncertainty'", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("topical overlap"); }); // Test 3 — independent resolvability means distinct uncertainty it("test 3: uncertainty that can remain unresolved after existing is resolved is distinct", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("remain unresolved"); }); // Test 4 — equivalent savings-uncertainty wording prefers reuse/refine (via existing-first) it("test 4: existing-first ordering ensures equivalent uncertainties are refined not duplicated", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("first check whether an existing unresolved node"); expect(guidance).toContain("if so, update/refine that existing structure rather than adding a duplicate"); }); // Test 5 — broad cost vs savings realism requires distinct unknown it("test 5: the guidance prevents broad nodes from automatically absorbing sub-concerns", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("not automatically"); }); // Test 6 — retention vs productivity domain overlap handled separately it("test 6: unrelated domains remain separate even if superficially topical", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("separately"); }); // Test 7 — duplicate avoidance preserved (not weakened) it("test 7: existing duplicate-avoidance remains intact", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("Do not add duplicate unknowns"); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes"); }); // Test 8 — existing-first ordering preserved (not displaced) it("test 8: existing-first ordering is intact in the assembled prompt", () => { const prompt = buildGraphUpdatePrompt(makeContext()); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("first check whether an existing unresolved node"); expect(guidance).toContain("if no such node exists, add a new unknown"); }); // Test 9 — no keyword/synonym/semantic-matching machinery added it("test 9: no new deterministic semantic matcher or keyword matching was added", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).not.toContain("threshold"); expect(prompt).not.toContain("synonym"); expect(prompt).not.toContain("keyword match"); expect(prompt).not.toContain("embedding"); expect(prompt).not.toContain("similarity"); }); // Test 10 — structured semantic fidelity preserved (supportCategory, resolutionGuidance) it("test 10: structured semantic fidelity instructions remain intact", () => { const prompt = buildGraphUpdatePrompt(makeContext()); expect(prompt).toContain("supportCategory"); expect(prompt).toContain("resolutionGuidance"); expect(prompt).toContain("genuinely new concepts"); const guidance = getAdditionalGuidance(prompt); expect(guidance).toContain("preserves semantic fidelity"); }); });