reasoning: add proposal-level answer meaning guard

- Add answerMeaning schema with supportCategory and resolutionGuidance enums
- Add pre-mutation guard that validates proposal alignment with answerMeaning
- Update prompt builder to instruct the model on answerMeaning contract
- Add tests for schema, guard logic, parsing defaults, and regression cases A-D
This commit is contained in:
2026-08-09 07:07:21 +01:00
parent 8c1036ecd0
commit 0d7ad5775c
7 changed files with 470 additions and 0 deletions
+134
View File
@@ -2682,6 +2682,137 @@ function answerConfirmsComparability(answer) {
);
}
function normaliseSemanticText(value) {
return String(value || "")
.toLowerCase()
.replace(/\s+/g, " ")
.trim();
}
function hasConditionalQualification(text) {
const value = normaliseSemanticText(text);
return (
value.includes("might") ||
value.includes("normally") ||
value.includes("for the right opportunity") ||
value.includes("depends") ||
value.includes("conditional") ||
value.includes("under specific")
);
}
function containsConstraintBoundaryLanguage(text) {
const value = normaliseSemanticText(text);
return (
value.includes("hard constraint") ||
value.includes("constraint") ||
value.includes("non negotiable") ||
value.includes("non-negotiable") ||
value.includes("preference") ||
value.includes("trade off") ||
value.includes("trade-off")
);
}
function containsWeakeningOfHardConstraint(text) {
const value = normaliseSemanticText(text);
return (
value.includes("preference") ||
value.includes("trade off") ||
value.includes("trade-off") ||
value.includes("not a hard constraint") ||
value.includes("rather than a hard constraint")
);
}
function proposalResolutionSummary(proposal) {
const resolved =
proposal.resolvedUnknownNodeIds.length > 0 ||
proposal.updatedNodes.some((update) => update.newStatus === "resolved");
const proposalText = normaliseSemanticText(
[
...(proposal.updatedNodes || []).flatMap((update) => [
update.reason,
update.newValue,
]),
proposal.selectedQuestion?.question,
proposal.selectedQuestion?.reason,
]
.filter(Boolean)
.join(" "),
);
return { resolved, proposalText };
}
function validateAnswerMeaningAlignment(proposal) {
if (!proposal.answerMeaning) return [];
const errors = [];
const { userSupportedMeaning, supportCategory, resolutionGuidance } =
proposal.answerMeaning;
const meaningText = normaliseSemanticText(userSupportedMeaning);
const { resolved, proposalText } = proposalResolutionSummary(proposal);
if (resolutionGuidance === "must_remain_unresolved" && resolved) {
errors.push(
"Proposal resolves an unknown even though answerMeaning says the user's answer must remain unresolved.",
);
}
if (supportCategory === "relative_priority_only") {
if (containsConstraintBoundaryLanguage(meaningText)) {
errors.push(
"answerMeaning.userSupportedMeaning for relative_priority_only must not introduce a constraint or preference/trade-off judgement.",
);
}
if (containsConstraintBoundaryLanguage(proposalText)) {
errors.push(
"Proposal introduces unsupported constraint-boundary interpretation from a relative-priority answer.",
);
}
}
if (supportCategory === "conditional_tradeoff") {
if (!hasConditionalQualification(meaningText)) {
errors.push(
"answerMeaning.userSupportedMeaning for conditional_tradeoff must preserve the user's qualification or condition.",
);
}
if (resolved && !hasConditionalQualification(proposalText)) {
errors.push(
"Proposal resolves a conditional trade-off without preserving its conditional qualification in the proposed change.",
);
}
}
if (supportCategory === "uncertain") {
if (containsConstraintBoundaryLanguage(proposalText)) {
errors.push(
"Proposal introduces a stronger interpretation even though answerMeaning marks the answer as uncertain.",
);
}
}
if (supportCategory === "explicit_hard_constraint") {
if (!resolved && resolutionGuidance === "must_resolve") {
errors.push(
"Proposal leaves an explicitly stated hard constraint unresolved.",
);
}
if (containsWeakeningOfHardConstraint(proposalText)) {
errors.push(
"Proposal weakens an explicitly stated hard constraint into a preference or trade-off.",
);
}
}
return errors;
}
function deriveReasoningStateOverride({
graph,
previousQuestion,
@@ -2853,6 +2984,9 @@ export function applyValidatedProposal({
validatedProposal,
);
proposalCompatibilityErrors.push(...selectedQuestionValidation.errors);
proposalCompatibilityErrors.push(
...validateAnswerMeaningAlignment(validatedProposal),
);
proposalCompatibilityErrors.push(
...validateQuestionSelectionRequirement(situationGraph, validatedProposal),
);
+10
View File
@@ -69,6 +69,7 @@ The JSON object must contain exactly these top-level fields:
- resolvedUnknownNodeIds
- affectedNodeIds
- selectedQuestion
- answerMeaning
## Required Shapes
- addedNodes: array of nodes using these exact keys:
@@ -82,6 +83,8 @@ The JSON object must contain exactly these top-level fields:
- affectedNodeIds: array of strings
- selectedQuestion: either null or an object using these exact keys:
nodeId, question, reason
- answerMeaning: either null or an object using these exact keys:
userSupportedMeaning, possibleInference, supportCategory, resolutionGuidance
## Proposal Rules
1. Propose changes only. Never return a replacement graph.
@@ -111,6 +114,12 @@ The JSON object must contain exactly these top-level fields:
23. Never use unknown enum values.
24. Do not change existing IDs.
25. Do not replace the whole graph, and do not restate unchanged graph content inside the proposal.
26. answerMeaning.userSupportedMeaning must state only what the user's answer directly supports.
27. Put any stronger interpretation in answerMeaning.possibleInference, not in userSupportedMeaning.
28. If the answer is only a relative priority statement, use supportCategory=relative_priority_only and resolutionGuidance=must_remain_unresolved.
29. If the answer is conditional or qualified, preserve that qualification explicitly in userSupportedMeaning.
30. If the answer says the user is unsure or does not resolve the distinction, use supportCategory=uncertain and resolutionGuidance=must_remain_unresolved.
31. If the answer explicitly states a hard constraint, use supportCategory=explicit_hard_constraint and resolutionGuidance=must_resolve.
## Additional Guidance
- If the answer only clarifies an existing unknown, prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes.
@@ -120,6 +129,7 @@ The JSON object must contain exactly these top-level fields:
- If you add a new unknown, its description must do two jobs in one sentence: what is unknown, and why resolving it matters for the case.
- Treat selectedQuestion as a candidate only; the engine will apply deterministic information-value scoring after validation.
- If the answer does not justify a change, return empty arrays for every category.
- Use answerMeaning to preserve the answer's direct meaning even when the graph change remains unresolved.
## Example Constraint Reminder
${formatExampleAnswerBlock()}
+24
View File
@@ -144,6 +144,29 @@ const graphUpdateNodeChangeSchema = z.object({
reason: z.string().min(1),
});
export const answerSupportCategory = /** @type {const} */ ({
relative_priority_only: "relative_priority_only",
conditional_tradeoff: "conditional_tradeoff",
uncertain: "uncertain",
explicit_hard_constraint: "explicit_hard_constraint",
other: "other",
});
export const answerResolutionGuidance = /** @type {const} */ ({
must_remain_unresolved: "must_remain_unresolved",
may_resolve: "may_resolve",
must_resolve: "must_resolve",
});
export const answerMeaningSchema = z
.object({
userSupportedMeaning: z.string().min(1),
possibleInference: z.string().nullable().optional(),
supportCategory: z.enum(Object.values(answerSupportCategory)),
resolutionGuidance: z.enum(Object.values(answerResolutionGuidance)),
})
.strict();
export const selectedQuestionSchema = z
.object({
nodeId: z.string().min(1),
@@ -160,6 +183,7 @@ export const graphUpdateSchema = z.object({
resolvedUnknownNodeIds: z.array(z.string()).default([]),
affectedNodeIds: z.array(z.string()).default([]),
selectedQuestion: selectedQuestionSchema.nullable().default(null),
answerMeaning: answerMeaningSchema.nullable().default(null),
});
/** @typedef {z.infer<typeof graphUpdateSchema>} GraphUpdate */
+240
View File
@@ -254,6 +254,30 @@ function makeCommercialUpdateFixture() {
});
}
function makeRiskClarificationFixture() {
const riskUnknown = makeNode({
id: "n-risk-constraint",
label: "Whether avoiding more risk is a hard constraint",
description:
"Need to know whether avoiding additional risk is a hard constraint or a preference/trade-off.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const graph = makeGraph({
centralStatement:
"I want the business to grow, but I don't want to take on more risk.",
nodes: [riskUnknown],
edges: [],
activeUnknownNodeId: riskUnknown.id,
resolvedNodeIds: [],
currentSummary: "Risk clarification fixture",
});
return { graph, riskUnknownId: riskUnknown.id };
}
function makeMeaningfulNoOpProposal() {
return {
addedNodes: [
@@ -1012,6 +1036,222 @@ describe("applyValidatedProposal", () => {
);
});
it("Regression A: rejects weak priority being strengthened into a resolved constraint judgement", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "Risk matters more to me.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Risk avoidance is not a hard constraint; it is a stronger priority.",
reason:
"The answer implies risk matters more but is not a hard constraint.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"Risk is of greater relative importance than growth.",
possibleInference:
"This may imply caution, but does not establish whether risk is a hard constraint.",
supportCategory: "relative_priority_only",
resolutionGuidance: "must_remain_unresolved",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain("must remain unresolved");
});
it("Regression B: rejects conditional trade-off proposals that flatten the qualification", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer:
"I'd normally avoid more risk, but for the right opportunity I might accept some.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Avoiding additional risk is a preference or trade-off rather than a hard constraint.",
reason:
"The answer shows a preference or trade-off rather than a hard constraint.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"The user would normally avoid more risk, but for the right opportunity might accept some.",
possibleInference:
"This may support eventual clarification, but the qualifying condition remains material.",
supportCategory: "conditional_tradeoff",
resolutionGuidance: "may_resolve",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain("conditional qualification");
});
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "I'm not really sure.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Risk avoidance is probably a preference rather than a hard constraint.",
reason:
"The answer suggests uncertainty but leans toward preference.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"The user is not sure whether avoiding additional risk is a hard constraint or a preference/trade-off.",
possibleInference: null,
supportCategory: "uncertain",
resolutionGuidance: "must_remain_unresolved",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain("must remain unresolved");
});
it("Regression D: rejects weakening an explicit hard constraint", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "It's a hard constraint. I don't want any increase in risk.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Avoiding additional risk is a preference or trade-off rather than a hard constraint.",
reason:
"The answer was interpreted as a strong preference rather than a hard constraint.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"Avoiding additional risk is a hard constraint and the user does not want any increase in risk.",
possibleInference: null,
supportCategory: "explicit_hard_constraint",
resolutionGuidance: "must_resolve",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain(
"weakens an explicitly stated hard constraint",
);
});
it("accepts explicit hard constraint when proposal preserves it", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "It's a hard constraint. I don't want any increase in risk.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Avoiding additional risk is a hard constraint. The user does not want any increase in risk.",
reason:
"The answer explicitly states a hard constraint with no allowed increase in risk.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"Avoiding additional risk is a hard constraint and the user does not want any increase in risk.",
possibleInference: null,
supportCategory: "explicit_hard_constraint",
resolutionGuidance: "must_resolve",
},
},
});
expect(result.success).toBe(true);
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(
riskUnknownId,
);
});
it("active unknown matches selected question node", () => {
const { graph, ids } = makeApplicationFixture();
+14
View File
@@ -73,6 +73,7 @@ describe("buildGraphUpdatePrompt", () => {
expect(prompt).toContain("resolvedUnknownNodeIds");
expect(prompt).toContain("affectedNodeIds");
expect(prompt).toContain("selectedQuestion");
expect(prompt).toContain("answerMeaning");
});
it("lists enum values", () => {
@@ -111,4 +112,17 @@ describe("buildGraphUpdatePrompt", () => {
"the engine will deterministically choose final priority after validation",
);
});
it("instructs the model to preserve user-supported meaning separately from inference", () => {
const prompt = buildGraphUpdatePrompt(makeContext());
expect(prompt).toContain(
"answerMeaning.userSupportedMeaning must state only what the user's answer directly supports",
);
expect(prompt).toContain(
"Put any stronger interpretation in answerMeaning.possibleInference",
);
expect(prompt).toContain(
"If the answer is only a relative priority statement, use supportCategory=relative_priority_only and resolutionGuidance=must_remain_unresolved",
);
});
});
+27
View File
@@ -181,6 +181,13 @@ describe("graphUpdateSchema", () => {
question: "What does this new node mean?",
reason: "A follow-up unknown remains.",
},
answerMeaning: {
userSupportedMeaning:
"The user directly established a concrete answer.",
possibleInference: null,
supportCategory: "other",
resolutionGuidance: "may_resolve",
},
});
expect(result.success).toBe(true);
});
@@ -192,6 +199,26 @@ describe("graphUpdateSchema", () => {
expect(result.success).toBe(true);
});
it("allows null answerMeaning", () => {
const result = graphUpdateSchema.safeParse({
answerMeaning: null,
});
expect(result.success).toBe(true);
});
it("validates structured answerMeaning when present", () => {
const result = graphUpdateSchema.safeParse({
answerMeaning: {
userSupportedMeaning: "Risk matters more to me.",
possibleInference:
"This may imply caution, but does not establish a hard constraint.",
supportCategory: "relative_priority_only",
resolutionGuidance: "must_remain_unresolved",
},
});
expect(result.success).toBe(true);
});
it("rejects update with invalid node kind in addedNodes", () => {
const invalid = graphUpdateSchema.safeParse({
addedNodes: [
+21
View File
@@ -19,6 +19,13 @@ function makeValidProposal(overrides = {}) {
resolvedUnknownNodeIds: ["n-unknown"],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"The user directly provided the updated complaint rate.",
possibleInference: null,
supportCategory: "other",
resolutionGuidance: "may_resolve",
},
...overrides,
};
}
@@ -131,6 +138,20 @@ describe("parseGraphUpdateProposal", () => {
expect(result.proposal.selectedQuestion).toBeNull();
});
it("defaults missing answerMeaning to null", () => {
const result = parseGraphUpdateProposal({
addedNodes: [],
updatedNodes: [],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [],
affectedNodeIds: [],
selectedQuestion: null,
});
expect(result.success).toBe(true);
expect(result.proposal.answerMeaning).toBeNull();
});
it("parses a valid selectedQuestion", () => {
const result = parseGraphUpdateProposal(
makeValidProposal({