reasoning: add proposal-level answer meaning guard
- Add answerMeaning schema with supportCategory and resolutionGuidance enums - Add pre-mutation guard that validates proposal alignment with answerMeaning - Update prompt builder to instruct the model on answerMeaning contract - Add tests for schema, guard logic, parsing defaults, and regression cases A-D
This commit is contained in:
@@ -254,6 +254,30 @@ function makeCommercialUpdateFixture() {
|
||||
});
|
||||
}
|
||||
|
||||
function makeRiskClarificationFixture() {
|
||||
const riskUnknown = makeNode({
|
||||
id: "n-risk-constraint",
|
||||
label: "Whether avoiding more risk is a hard constraint",
|
||||
description:
|
||||
"Need to know whether avoiding additional risk is a hard constraint or a preference/trade-off.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"I want the business to grow, but I don't want to take on more risk.",
|
||||
nodes: [riskUnknown],
|
||||
edges: [],
|
||||
activeUnknownNodeId: riskUnknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Risk clarification fixture",
|
||||
});
|
||||
|
||||
return { graph, riskUnknownId: riskUnknown.id };
|
||||
}
|
||||
|
||||
function makeMeaningfulNoOpProposal() {
|
||||
return {
|
||||
addedNodes: [
|
||||
@@ -1012,6 +1036,222 @@ describe("applyValidatedProposal", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("Regression A: rejects weak priority being strengthened into a resolved constraint judgement", () => {
|
||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer: "Risk matters more to me.",
|
||||
previousQuestion:
|
||||
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: riskUnknownId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"Risk avoidance is not a hard constraint; it is a stronger priority.",
|
||||
reason:
|
||||
"The answer implies risk matters more but is not a hard constraint.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [riskUnknownId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"Risk is of greater relative importance than growth.",
|
||||
possibleInference:
|
||||
"This may imply caution, but does not establish whether risk is a hard constraint.",
|
||||
supportCategory: "relative_priority_only",
|
||||
resolutionGuidance: "must_remain_unresolved",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors.join(" ")).toContain("must remain unresolved");
|
||||
});
|
||||
|
||||
it("Regression B: rejects conditional trade-off proposals that flatten the qualification", () => {
|
||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer:
|
||||
"I'd normally avoid more risk, but for the right opportunity I might accept some.",
|
||||
previousQuestion:
|
||||
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: riskUnknownId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"Avoiding additional risk is a preference or trade-off rather than a hard constraint.",
|
||||
reason:
|
||||
"The answer shows a preference or trade-off rather than a hard constraint.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [riskUnknownId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"The user would normally avoid more risk, but for the right opportunity might accept some.",
|
||||
possibleInference:
|
||||
"This may support eventual clarification, but the qualifying condition remains material.",
|
||||
supportCategory: "conditional_tradeoff",
|
||||
resolutionGuidance: "may_resolve",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors.join(" ")).toContain("conditional qualification");
|
||||
});
|
||||
|
||||
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
|
||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer: "I'm not really sure.",
|
||||
previousQuestion:
|
||||
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: riskUnknownId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"Risk avoidance is probably a preference rather than a hard constraint.",
|
||||
reason:
|
||||
"The answer suggests uncertainty but leans toward preference.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [riskUnknownId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"The user is not sure whether avoiding additional risk is a hard constraint or a preference/trade-off.",
|
||||
possibleInference: null,
|
||||
supportCategory: "uncertain",
|
||||
resolutionGuidance: "must_remain_unresolved",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors.join(" ")).toContain("must remain unresolved");
|
||||
});
|
||||
|
||||
it("Regression D: rejects weakening an explicit hard constraint", () => {
|
||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer: "It's a hard constraint. I don't want any increase in risk.",
|
||||
previousQuestion:
|
||||
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: riskUnknownId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"Avoiding additional risk is a preference or trade-off rather than a hard constraint.",
|
||||
reason:
|
||||
"The answer was interpreted as a strong preference rather than a hard constraint.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [riskUnknownId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"Avoiding additional risk is a hard constraint and the user does not want any increase in risk.",
|
||||
possibleInference: null,
|
||||
supportCategory: "explicit_hard_constraint",
|
||||
resolutionGuidance: "must_resolve",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors.join(" ")).toContain(
|
||||
"weakens an explicitly stated hard constraint",
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts explicit hard constraint when proposal preserves it", () => {
|
||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer: "It's a hard constraint. I don't want any increase in risk.",
|
||||
previousQuestion:
|
||||
"Is avoiding additional risk a hard constraint or a preference/trade-off?",
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: riskUnknownId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"Avoiding additional risk is a hard constraint. The user does not want any increase in risk.",
|
||||
reason:
|
||||
"The answer explicitly states a hard constraint with no allowed increase in risk.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [riskUnknownId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"Avoiding additional risk is a hard constraint and the user does not want any increase in risk.",
|
||||
possibleInference: null,
|
||||
supportCategory: "explicit_hard_constraint",
|
||||
resolutionGuidance: "must_resolve",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(
|
||||
riskUnknownId,
|
||||
);
|
||||
});
|
||||
|
||||
it("active unknown matches selected question node", () => {
|
||||
const { graph, ids } = makeApplicationFixture();
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ describe("buildGraphUpdatePrompt", () => {
|
||||
expect(prompt).toContain("resolvedUnknownNodeIds");
|
||||
expect(prompt).toContain("affectedNodeIds");
|
||||
expect(prompt).toContain("selectedQuestion");
|
||||
expect(prompt).toContain("answerMeaning");
|
||||
});
|
||||
|
||||
it("lists enum values", () => {
|
||||
@@ -111,4 +112,17 @@ describe("buildGraphUpdatePrompt", () => {
|
||||
"the engine will deterministically choose final priority after validation",
|
||||
);
|
||||
});
|
||||
|
||||
it("instructs the model to preserve user-supported meaning separately from inference", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"answerMeaning.userSupportedMeaning must state only what the user's answer directly supports",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"Put any stronger interpretation in answerMeaning.possibleInference",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"If the answer is only a relative priority statement, use supportCategory=relative_priority_only and resolutionGuidance=must_remain_unresolved",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -181,6 +181,13 @@ describe("graphUpdateSchema", () => {
|
||||
question: "What does this new node mean?",
|
||||
reason: "A follow-up unknown remains.",
|
||||
},
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"The user directly established a concrete answer.",
|
||||
possibleInference: null,
|
||||
supportCategory: "other",
|
||||
resolutionGuidance: "may_resolve",
|
||||
},
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
@@ -192,6 +199,26 @@ describe("graphUpdateSchema", () => {
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows null answerMeaning", () => {
|
||||
const result = graphUpdateSchema.safeParse({
|
||||
answerMeaning: null,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates structured answerMeaning when present", () => {
|
||||
const result = graphUpdateSchema.safeParse({
|
||||
answerMeaning: {
|
||||
userSupportedMeaning: "Risk matters more to me.",
|
||||
possibleInference:
|
||||
"This may imply caution, but does not establish a hard constraint.",
|
||||
supportCategory: "relative_priority_only",
|
||||
resolutionGuidance: "must_remain_unresolved",
|
||||
},
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects update with invalid node kind in addedNodes", () => {
|
||||
const invalid = graphUpdateSchema.safeParse({
|
||||
addedNodes: [
|
||||
|
||||
@@ -19,6 +19,13 @@ function makeValidProposal(overrides = {}) {
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"The user directly provided the updated complaint rate.",
|
||||
possibleInference: null,
|
||||
supportCategory: "other",
|
||||
resolutionGuidance: "may_resolve",
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -131,6 +138,20 @@ describe("parseGraphUpdateProposal", () => {
|
||||
expect(result.proposal.selectedQuestion).toBeNull();
|
||||
});
|
||||
|
||||
it("defaults missing answerMeaning to null", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
addedNodes: [],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.answerMeaning).toBeNull();
|
||||
});
|
||||
|
||||
it("parses a valid selectedQuestion", () => {
|
||||
const result = parseGraphUpdateProposal(
|
||||
makeValidProposal({
|
||||
|
||||
Reference in New Issue
Block a user