feat: define graph update proposal contract
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
import {
|
||||
ConfidenceLevel,
|
||||
SituationKind,
|
||||
SituationRelationship,
|
||||
SituationStatus,
|
||||
} from "./schema.js";
|
||||
|
||||
const DEFAULT_PROMPT_VERSION = "v0.4";
|
||||
|
||||
function formatEnumValues(values) {
|
||||
return Object.values(values).join(" | ");
|
||||
}
|
||||
|
||||
function formatGraph(graph) {
|
||||
return JSON.stringify(graph, null, 2);
|
||||
}
|
||||
|
||||
function formatExampleAnswerBlock() {
|
||||
return [
|
||||
"Example answer the model must be able to handle without hard-coding output:",
|
||||
'"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units."',
|
||||
"This may justify resolving a rate-related unknown or updating a metric node, but only if the current graph and answer support that proposal.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildGraphUpdatePrompt({
|
||||
situationGraph,
|
||||
previousQuestion,
|
||||
answer,
|
||||
promptVersion = DEFAULT_PROMPT_VERSION,
|
||||
}) {
|
||||
const nodeKinds = formatEnumValues(SituationKind);
|
||||
const nodeStatuses = formatEnumValues(SituationStatus);
|
||||
const edgeRelationships = formatEnumValues(SituationRelationship);
|
||||
const confidenceLevels = formatEnumValues(ConfidenceLevel);
|
||||
|
||||
return `You are proposing a graph update for Confidence Engine ${promptVersion}.
|
||||
|
||||
Return exactly one JSON object matching the GraphUpdate contract.
|
||||
Return JSON only. Do not include markdown, explanation, or any text before or after the JSON object.
|
||||
|
||||
## Current Situation Graph
|
||||
${formatGraph(situationGraph)}
|
||||
|
||||
## Previous Selected Question
|
||||
${previousQuestion}
|
||||
|
||||
## User Answer
|
||||
${answer}
|
||||
|
||||
## Allowed Node Kinds
|
||||
${nodeKinds}
|
||||
|
||||
## Allowed Node Statuses
|
||||
${nodeStatuses}
|
||||
|
||||
## Allowed Edge Relationships
|
||||
${edgeRelationships}
|
||||
|
||||
## Allowed Confidence Values
|
||||
${confidenceLevels}
|
||||
|
||||
## Required JSON Field Names
|
||||
The JSON object must contain exactly these top-level fields:
|
||||
- addedNodes
|
||||
- updatedNodes
|
||||
- addedEdges
|
||||
- removedEdgeIds
|
||||
- resolvedUnknownNodeIds
|
||||
- affectedNodeIds
|
||||
|
||||
## Required Shapes
|
||||
- addedNodes: array of nodes using these exact keys:
|
||||
id, label, description, kind, status, confidence, value, unit, evidenceIds, dependsOn, affects, parentId, childIds
|
||||
- updatedNodes: array of node updates using these exact keys:
|
||||
nodeId, previousStatus, newStatus, previousValue, newValue, reason
|
||||
- addedEdges: array of edges using these exact keys:
|
||||
id, fromNodeId, toNodeId, relationship, confidence, description
|
||||
- removedEdgeIds: array of strings
|
||||
- resolvedUnknownNodeIds: array of strings
|
||||
- affectedNodeIds: array of strings
|
||||
|
||||
## Proposal Rules
|
||||
1. Propose changes only. Never return a replacement graph.
|
||||
2. Preserve unrelated nodes and edges by omitting them from the proposal.
|
||||
3. Reference existing node IDs when updating an existing concept.
|
||||
4. Use addedNodes only for genuinely new concepts.
|
||||
5. Resolve the active unknown when the answer supports it.
|
||||
6. Propagate only through explicit dependencies or relationships already present in the graph.
|
||||
7. Do not invent evidence.
|
||||
8. Do not create unsupported causal edges.
|
||||
9. Do not ask more than one next question. In this contract you are not returning any next-question field at all.
|
||||
10. Use empty arrays when there are no changes in a category.
|
||||
11. Never return null array entries.
|
||||
12. Never use unknown enum values.
|
||||
13. Do not change existing IDs.
|
||||
14. Do not replace the whole graph, and do not restate unchanged graph content inside the proposal.
|
||||
|
||||
## Additional Guidance
|
||||
- If the answer only clarifies an existing unknown, prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes.
|
||||
- If a new metric or observation is necessary, add the smallest set of nodes and edges needed.
|
||||
- If the answer does not justify a change, return empty arrays for every category.
|
||||
|
||||
## Example Constraint Reminder
|
||||
${formatExampleAnswerBlock()}
|
||||
|
||||
## Output Contract Reminder
|
||||
Return one JSON object only, with exact field names and exact enum values.
|
||||
Never include a full graph.
|
||||
Never include a nextQuestion field.
|
||||
`;
|
||||
}
|
||||
|
||||
export const buildUpdatePrompt = buildGraphUpdatePrompt;
|
||||
@@ -0,0 +1,138 @@
|
||||
import { graphUpdateSchema } from "./schema.js";
|
||||
|
||||
const TOP_LEVEL_ARRAY_FIELDS = [
|
||||
"addedNodes",
|
||||
"updatedNodes",
|
||||
"addedEdges",
|
||||
"removedEdgeIds",
|
||||
"resolvedUnknownNodeIds",
|
||||
"affectedNodeIds",
|
||||
];
|
||||
|
||||
function cloneJsonSafe(value) {
|
||||
if (value == null) return value;
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
function removeNullArrayEntries(value, path = [], normalisationsApplied = []) {
|
||||
if (Array.isArray(value)) {
|
||||
const filtered = [];
|
||||
value.forEach((item, index) => {
|
||||
if (item === null) {
|
||||
normalisationsApplied.push({
|
||||
path: [...path, index],
|
||||
change: "Removed null array entry",
|
||||
});
|
||||
return;
|
||||
}
|
||||
filtered.push(
|
||||
removeNullArrayEntries(item, [...path, index], normalisationsApplied),
|
||||
);
|
||||
});
|
||||
return filtered;
|
||||
}
|
||||
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, child]) => [
|
||||
key,
|
||||
removeNullArrayEntries(child, [...path, key], normalisationsApplied),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function applyKnownEnumAliases(proposal, normalisationsApplied) {
|
||||
if (!proposal || typeof proposal !== "object") return proposal;
|
||||
|
||||
if (Array.isArray(proposal.addedNodes)) {
|
||||
proposal.addedNodes = proposal.addedNodes.map((node, index) => {
|
||||
if (node?.kind === "reported_statement") {
|
||||
normalisationsApplied.push({
|
||||
path: ["addedNodes", index, "kind"],
|
||||
change: "Converted reported_statement to reported_claim",
|
||||
});
|
||||
return { ...node, kind: "reported_claim" };
|
||||
}
|
||||
return node;
|
||||
});
|
||||
}
|
||||
|
||||
return proposal;
|
||||
}
|
||||
|
||||
function fillMissingOptionalArrays(proposal, normalisationsApplied) {
|
||||
if (!proposal || typeof proposal !== "object") return proposal;
|
||||
|
||||
for (const field of TOP_LEVEL_ARRAY_FIELDS) {
|
||||
if (!(field in proposal)) {
|
||||
proposal[field] = [];
|
||||
normalisationsApplied.push({
|
||||
path: [field],
|
||||
change: "Filled missing optional array with []",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return proposal;
|
||||
}
|
||||
|
||||
export function parseGraphUpdateProposal(rawResponse) {
|
||||
const raw = rawResponse;
|
||||
let parsed;
|
||||
|
||||
if (typeof rawResponse === "string") {
|
||||
try {
|
||||
parsed = JSON.parse(rawResponse);
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
proposal: null,
|
||||
raw,
|
||||
normalisationsApplied: [],
|
||||
errors: [error.message || "Model response is not valid JSON"],
|
||||
};
|
||||
}
|
||||
} else if (rawResponse && typeof rawResponse === "object") {
|
||||
parsed = cloneJsonSafe(rawResponse);
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
proposal: null,
|
||||
raw,
|
||||
normalisationsApplied: [],
|
||||
errors: ["Graph update proposal must be a JSON object or JSON string"],
|
||||
};
|
||||
}
|
||||
|
||||
const normalisationsApplied = [];
|
||||
let normalised = removeNullArrayEntries(parsed, [], normalisationsApplied);
|
||||
normalised = applyKnownEnumAliases(normalised, normalisationsApplied);
|
||||
normalised = fillMissingOptionalArrays(normalised, normalisationsApplied);
|
||||
|
||||
const parsedProposal = graphUpdateSchema.safeParse(normalised);
|
||||
|
||||
if (!parsedProposal.success) {
|
||||
return {
|
||||
success: false,
|
||||
proposal: null,
|
||||
raw,
|
||||
normalisationsApplied,
|
||||
errors: parsedProposal.error.issues.map((issue) => ({
|
||||
path: issue.path,
|
||||
message: issue.message,
|
||||
code: issue.code,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
proposal: parsedProposal.data,
|
||||
raw,
|
||||
normalisationsApplied,
|
||||
errors: [],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildGraphUpdatePrompt } from "@/lib/graph/prompt-builder.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeContext() {
|
||||
const unknown = makeNode({
|
||||
id: "n-unknown",
|
||||
label: "Complaint rate denominator",
|
||||
description: "Need the denominator to compare complaint rates",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const observation = makeNode({
|
||||
id: "n-obs",
|
||||
label: "Complaints up 35%",
|
||||
description: "Complaints increased by 35%",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
return {
|
||||
situationGraph: makeGraph({
|
||||
centralStatement: "Complaints increased while production increased.",
|
||||
nodes: [unknown, observation],
|
||||
edges: [
|
||||
makeEdge({
|
||||
id: "e1",
|
||||
fromNodeId: observation.id,
|
||||
toNodeId: unknown.id,
|
||||
relationship: "supports",
|
||||
confidence: "high",
|
||||
description: "Observation informs the unknown",
|
||||
}),
|
||||
],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary:
|
||||
"Nodes: 1 observation, 1 unknown | Edges: 1 total | Unknowns: 1 unresolved",
|
||||
}),
|
||||
previousQuestion: "What denominator is being used for the complaint rate?",
|
||||
answer:
|
||||
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildGraphUpdatePrompt", () => {
|
||||
it("includes the current graph", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"Complaints increased while production increased.",
|
||||
);
|
||||
expect(prompt).toContain("Complaint rate denominator");
|
||||
});
|
||||
|
||||
it("includes previous question and answer", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"What denominator is being used for the complaint rate?",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
||||
);
|
||||
});
|
||||
|
||||
it("contains exact schema keys", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("addedNodes");
|
||||
expect(prompt).toContain("updatedNodes");
|
||||
expect(prompt).toContain("addedEdges");
|
||||
expect(prompt).toContain("removedEdgeIds");
|
||||
expect(prompt).toContain("resolvedUnknownNodeIds");
|
||||
expect(prompt).toContain("affectedNodeIds");
|
||||
});
|
||||
|
||||
it("lists enum values", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"observation | reported_claim | metric | state | transition | relationship | assumption | unknown | conclusion",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"known | unknown | provisional | supported | weakened | contradicted | resolved",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"supports | weakens | contradicts | depends_on | causes | may_cause | measures | compares_with | updates | other",
|
||||
);
|
||||
});
|
||||
|
||||
it("forbids full-graph replacement", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("Never return a replacement graph");
|
||||
expect(prompt).toContain("Propose changes only");
|
||||
});
|
||||
|
||||
it("requires JSON only", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("Return JSON only");
|
||||
expect(prompt).toContain("Return one JSON object only");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseGraphUpdateProposal } from "@/lib/graph/update-proposal.js";
|
||||
|
||||
function makeValidProposal(overrides = {}) {
|
||||
return {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: "n-unknown",
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "1.9 complaints per 100 units",
|
||||
reason: "The answer directly provides the normalized complaint rate.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("parseGraphUpdateProposal", () => {
|
||||
it("parses a valid proposal", () => {
|
||||
const result = parseGraphUpdateProposal(makeValidProposal());
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.updatedNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("fails on malformed JSON", () => {
|
||||
const result = parseGraphUpdateProposal("{not json");
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("fails when required update content is invalid", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
updatedNodes: [{ nodeId: "n-unknown" }],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("removes null array entries and logs them", () => {
|
||||
const result = parseGraphUpdateProposal(
|
||||
JSON.stringify({
|
||||
...makeValidProposal(),
|
||||
addedNodes: [null],
|
||||
}),
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.addedNodes).toEqual([]);
|
||||
expect(result.normalisationsApplied).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ change: "Removed null array entry" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("fills missing optional arrays with empty arrays", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
updatedNodes: [],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.addedNodes).toEqual([]);
|
||||
expect(result.proposal.addedEdges).toEqual([]);
|
||||
expect(result.normalisationsApplied.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("normalises confirmed enum alias and preserves IDs", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
...makeValidProposal(),
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-new",
|
||||
label: "Reported update",
|
||||
description: "A new reported claim",
|
||||
kind: "reported_statement",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.addedNodes[0].kind).toBe("reported_claim");
|
||||
expect(result.proposal.addedNodes[0].id).toBe("n-new");
|
||||
});
|
||||
|
||||
it("unknown enum values still fail", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
...makeValidProposal(),
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-new",
|
||||
label: "Bad node",
|
||||
description: "Bad node",
|
||||
kind: "unsupported_kind",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("does not invent a next question", () => {
|
||||
const result = parseGraphUpdateProposal(makeValidProposal());
|
||||
expect(result.proposal.nextQuestion).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user