reasoning: add proposal-level answer meaning guard

- Add answerMeaning schema with supportCategory and resolutionGuidance enums
- Add pre-mutation guard that validates proposal alignment with answerMeaning
- Update prompt builder to instruct the model on answerMeaning contract
- Add tests for schema, guard logic, parsing defaults, and regression cases A-D
This commit is contained in:
2026-08-09 07:07:21 +01:00
parent 8c1036ecd0
commit 0d7ad5775c
7 changed files with 470 additions and 0 deletions
+240
View File
@@ -254,6 +254,30 @@ function makeCommercialUpdateFixture() {
});
}
function makeRiskClarificationFixture() {
const riskUnknown = makeNode({
id: "n-risk-constraint",
label: "Whether avoiding more risk is a hard constraint",
description:
"Need to know whether avoiding additional risk is a hard constraint or a preference/trade-off.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const graph = makeGraph({
centralStatement:
"I want the business to grow, but I don't want to take on more risk.",
nodes: [riskUnknown],
edges: [],
activeUnknownNodeId: riskUnknown.id,
resolvedNodeIds: [],
currentSummary: "Risk clarification fixture",
});
return { graph, riskUnknownId: riskUnknown.id };
}
function makeMeaningfulNoOpProposal() {
return {
addedNodes: [
@@ -1012,6 +1036,222 @@ describe("applyValidatedProposal", () => {
);
});
it("Regression A: rejects weak priority being strengthened into a resolved constraint judgement", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "Risk matters more to me.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Risk avoidance is not a hard constraint; it is a stronger priority.",
reason:
"The answer implies risk matters more but is not a hard constraint.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"Risk is of greater relative importance than growth.",
possibleInference:
"This may imply caution, but does not establish whether risk is a hard constraint.",
supportCategory: "relative_priority_only",
resolutionGuidance: "must_remain_unresolved",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain("must remain unresolved");
});
it("Regression B: rejects conditional trade-off proposals that flatten the qualification", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer:
"I'd normally avoid more risk, but for the right opportunity I might accept some.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Avoiding additional risk is a preference or trade-off rather than a hard constraint.",
reason:
"The answer shows a preference or trade-off rather than a hard constraint.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"The user would normally avoid more risk, but for the right opportunity might accept some.",
possibleInference:
"This may support eventual clarification, but the qualifying condition remains material.",
supportCategory: "conditional_tradeoff",
resolutionGuidance: "may_resolve",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain("conditional qualification");
});
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "I'm not really sure.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Risk avoidance is probably a preference rather than a hard constraint.",
reason:
"The answer suggests uncertainty but leans toward preference.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"The user is not sure whether avoiding additional risk is a hard constraint or a preference/trade-off.",
possibleInference: null,
supportCategory: "uncertain",
resolutionGuidance: "must_remain_unresolved",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain("must remain unresolved");
});
it("Regression D: rejects weakening an explicit hard constraint", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "It's a hard constraint. I don't want any increase in risk.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Avoiding additional risk is a preference or trade-off rather than a hard constraint.",
reason:
"The answer was interpreted as a strong preference rather than a hard constraint.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"Avoiding additional risk is a hard constraint and the user does not want any increase in risk.",
possibleInference: null,
supportCategory: "explicit_hard_constraint",
resolutionGuidance: "must_resolve",
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain(
"weakens an explicitly stated hard constraint",
);
});
it("accepts explicit hard constraint when proposal preserves it", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
answer: "It's a hard constraint. I don't want any increase in risk.",
previousQuestion:
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: riskUnknownId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"Avoiding additional risk is a hard constraint. The user does not want any increase in risk.",
reason:
"The answer explicitly states a hard constraint with no allowed increase in risk.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [riskUnknownId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"Avoiding additional risk is a hard constraint and the user does not want any increase in risk.",
possibleInference: null,
supportCategory: "explicit_hard_constraint",
resolutionGuidance: "must_resolve",
},
},
});
expect(result.success).toBe(true);
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(
riskUnknownId,
);
});
it("active unknown matches selected question node", () => {
const { graph, ids } = makeApplicationFixture();
+14
View File
@@ -73,6 +73,7 @@ describe("buildGraphUpdatePrompt", () => {
expect(prompt).toContain("resolvedUnknownNodeIds");
expect(prompt).toContain("affectedNodeIds");
expect(prompt).toContain("selectedQuestion");
expect(prompt).toContain("answerMeaning");
});
it("lists enum values", () => {
@@ -111,4 +112,17 @@ describe("buildGraphUpdatePrompt", () => {
"the engine will deterministically choose final priority after validation",
);
});
it("instructs the model to preserve user-supported meaning separately from inference", () => {
const prompt = buildGraphUpdatePrompt(makeContext());
expect(prompt).toContain(
"answerMeaning.userSupportedMeaning must state only what the user's answer directly supports",
);
expect(prompt).toContain(
"Put any stronger interpretation in answerMeaning.possibleInference",
);
expect(prompt).toContain(
"If the answer is only a relative priority statement, use supportCategory=relative_priority_only and resolutionGuidance=must_remain_unresolved",
);
});
});
+27
View File
@@ -181,6 +181,13 @@ describe("graphUpdateSchema", () => {
question: "What does this new node mean?",
reason: "A follow-up unknown remains.",
},
answerMeaning: {
userSupportedMeaning:
"The user directly established a concrete answer.",
possibleInference: null,
supportCategory: "other",
resolutionGuidance: "may_resolve",
},
});
expect(result.success).toBe(true);
});
@@ -192,6 +199,26 @@ describe("graphUpdateSchema", () => {
expect(result.success).toBe(true);
});
it("allows null answerMeaning", () => {
const result = graphUpdateSchema.safeParse({
answerMeaning: null,
});
expect(result.success).toBe(true);
});
it("validates structured answerMeaning when present", () => {
const result = graphUpdateSchema.safeParse({
answerMeaning: {
userSupportedMeaning: "Risk matters more to me.",
possibleInference:
"This may imply caution, but does not establish a hard constraint.",
supportCategory: "relative_priority_only",
resolutionGuidance: "must_remain_unresolved",
},
});
expect(result.success).toBe(true);
});
it("rejects update with invalid node kind in addedNodes", () => {
const invalid = graphUpdateSchema.safeParse({
addedNodes: [
+21
View File
@@ -19,6 +19,13 @@ function makeValidProposal(overrides = {}) {
resolvedUnknownNodeIds: ["n-unknown"],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"The user directly provided the updated complaint rate.",
possibleInference: null,
supportCategory: "other",
resolutionGuidance: "may_resolve",
},
...overrides,
};
}
@@ -131,6 +138,20 @@ describe("parseGraphUpdateProposal", () => {
expect(result.proposal.selectedQuestion).toBeNull();
});
it("defaults missing answerMeaning to null", () => {
const result = parseGraphUpdateProposal({
addedNodes: [],
updatedNodes: [],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [],
affectedNodeIds: [],
selectedQuestion: null,
});
expect(result.success).toBe(true);
expect(result.proposal.answerMeaning).toBeNull();
});
it("parses a valid selectedQuestion", () => {
const result = parseGraphUpdateProposal(
makeValidProposal({