reasoning: add proposal-level answer meaning guard
- Add answerMeaning schema with supportCategory and resolutionGuidance enums - Add pre-mutation guard that validates proposal alignment with answerMeaning - Update prompt builder to instruct the model on answerMeaning contract - Add tests for schema, guard logic, parsing defaults, and regression cases A-D
This commit is contained in:
@@ -2682,6 +2682,137 @@ function answerConfirmsComparability(answer) {
|
||||
);
|
||||
}
|
||||
|
||||
function normaliseSemanticText(value) {
|
||||
return String(value || "")
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function hasConditionalQualification(text) {
|
||||
const value = normaliseSemanticText(text);
|
||||
return (
|
||||
value.includes("might") ||
|
||||
value.includes("normally") ||
|
||||
value.includes("for the right opportunity") ||
|
||||
value.includes("depends") ||
|
||||
value.includes("conditional") ||
|
||||
value.includes("under specific")
|
||||
);
|
||||
}
|
||||
|
||||
function containsConstraintBoundaryLanguage(text) {
|
||||
const value = normaliseSemanticText(text);
|
||||
return (
|
||||
value.includes("hard constraint") ||
|
||||
value.includes("constraint") ||
|
||||
value.includes("non negotiable") ||
|
||||
value.includes("non-negotiable") ||
|
||||
value.includes("preference") ||
|
||||
value.includes("trade off") ||
|
||||
value.includes("trade-off")
|
||||
);
|
||||
}
|
||||
|
||||
function containsWeakeningOfHardConstraint(text) {
|
||||
const value = normaliseSemanticText(text);
|
||||
return (
|
||||
value.includes("preference") ||
|
||||
value.includes("trade off") ||
|
||||
value.includes("trade-off") ||
|
||||
value.includes("not a hard constraint") ||
|
||||
value.includes("rather than a hard constraint")
|
||||
);
|
||||
}
|
||||
|
||||
function proposalResolutionSummary(proposal) {
|
||||
const resolved =
|
||||
proposal.resolvedUnknownNodeIds.length > 0 ||
|
||||
proposal.updatedNodes.some((update) => update.newStatus === "resolved");
|
||||
const proposalText = normaliseSemanticText(
|
||||
[
|
||||
...(proposal.updatedNodes || []).flatMap((update) => [
|
||||
update.reason,
|
||||
update.newValue,
|
||||
]),
|
||||
proposal.selectedQuestion?.question,
|
||||
proposal.selectedQuestion?.reason,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" "),
|
||||
);
|
||||
|
||||
return { resolved, proposalText };
|
||||
}
|
||||
|
||||
function validateAnswerMeaningAlignment(proposal) {
|
||||
if (!proposal.answerMeaning) return [];
|
||||
|
||||
const errors = [];
|
||||
const { userSupportedMeaning, supportCategory, resolutionGuidance } =
|
||||
proposal.answerMeaning;
|
||||
const meaningText = normaliseSemanticText(userSupportedMeaning);
|
||||
const { resolved, proposalText } = proposalResolutionSummary(proposal);
|
||||
|
||||
if (resolutionGuidance === "must_remain_unresolved" && resolved) {
|
||||
errors.push(
|
||||
"Proposal resolves an unknown even though answerMeaning says the user's answer must remain unresolved.",
|
||||
);
|
||||
}
|
||||
|
||||
if (supportCategory === "relative_priority_only") {
|
||||
if (containsConstraintBoundaryLanguage(meaningText)) {
|
||||
errors.push(
|
||||
"answerMeaning.userSupportedMeaning for relative_priority_only must not introduce a constraint or preference/trade-off judgement.",
|
||||
);
|
||||
}
|
||||
|
||||
if (containsConstraintBoundaryLanguage(proposalText)) {
|
||||
errors.push(
|
||||
"Proposal introduces unsupported constraint-boundary interpretation from a relative-priority answer.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (supportCategory === "conditional_tradeoff") {
|
||||
if (!hasConditionalQualification(meaningText)) {
|
||||
errors.push(
|
||||
"answerMeaning.userSupportedMeaning for conditional_tradeoff must preserve the user's qualification or condition.",
|
||||
);
|
||||
}
|
||||
|
||||
if (resolved && !hasConditionalQualification(proposalText)) {
|
||||
errors.push(
|
||||
"Proposal resolves a conditional trade-off without preserving its conditional qualification in the proposed change.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (supportCategory === "uncertain") {
|
||||
if (containsConstraintBoundaryLanguage(proposalText)) {
|
||||
errors.push(
|
||||
"Proposal introduces a stronger interpretation even though answerMeaning marks the answer as uncertain.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (supportCategory === "explicit_hard_constraint") {
|
||||
if (!resolved && resolutionGuidance === "must_resolve") {
|
||||
errors.push(
|
||||
"Proposal leaves an explicitly stated hard constraint unresolved.",
|
||||
);
|
||||
}
|
||||
|
||||
if (containsWeakeningOfHardConstraint(proposalText)) {
|
||||
errors.push(
|
||||
"Proposal weakens an explicitly stated hard constraint into a preference or trade-off.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function deriveReasoningStateOverride({
|
||||
graph,
|
||||
previousQuestion,
|
||||
@@ -2853,6 +2984,9 @@ export function applyValidatedProposal({
|
||||
validatedProposal,
|
||||
);
|
||||
proposalCompatibilityErrors.push(...selectedQuestionValidation.errors);
|
||||
proposalCompatibilityErrors.push(
|
||||
...validateAnswerMeaningAlignment(validatedProposal),
|
||||
);
|
||||
proposalCompatibilityErrors.push(
|
||||
...validateQuestionSelectionRequirement(situationGraph, validatedProposal),
|
||||
);
|
||||
|
||||
@@ -69,6 +69,7 @@ The JSON object must contain exactly these top-level fields:
|
||||
- resolvedUnknownNodeIds
|
||||
- affectedNodeIds
|
||||
- selectedQuestion
|
||||
- answerMeaning
|
||||
|
||||
## Required Shapes
|
||||
- addedNodes: array of nodes using these exact keys:
|
||||
@@ -82,6 +83,8 @@ The JSON object must contain exactly these top-level fields:
|
||||
- affectedNodeIds: array of strings
|
||||
- selectedQuestion: either null or an object using these exact keys:
|
||||
nodeId, question, reason
|
||||
- answerMeaning: either null or an object using these exact keys:
|
||||
userSupportedMeaning, possibleInference, supportCategory, resolutionGuidance
|
||||
|
||||
## Proposal Rules
|
||||
1. Propose changes only. Never return a replacement graph.
|
||||
@@ -111,6 +114,12 @@ The JSON object must contain exactly these top-level fields:
|
||||
23. Never use unknown enum values.
|
||||
24. Do not change existing IDs.
|
||||
25. Do not replace the whole graph, and do not restate unchanged graph content inside the proposal.
|
||||
26. answerMeaning.userSupportedMeaning must state only what the user's answer directly supports.
|
||||
27. Put any stronger interpretation in answerMeaning.possibleInference, not in userSupportedMeaning.
|
||||
28. If the answer is only a relative priority statement, use supportCategory=relative_priority_only and resolutionGuidance=must_remain_unresolved.
|
||||
29. If the answer is conditional or qualified, preserve that qualification explicitly in userSupportedMeaning.
|
||||
30. If the answer says the user is unsure or does not resolve the distinction, use supportCategory=uncertain and resolutionGuidance=must_remain_unresolved.
|
||||
31. If the answer explicitly states a hard constraint, use supportCategory=explicit_hard_constraint and resolutionGuidance=must_resolve.
|
||||
|
||||
## Additional Guidance
|
||||
- If the answer only clarifies an existing unknown, prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes.
|
||||
@@ -120,6 +129,7 @@ The JSON object must contain exactly these top-level fields:
|
||||
- If you add a new unknown, its description must do two jobs in one sentence: what is unknown, and why resolving it matters for the case.
|
||||
- Treat selectedQuestion as a candidate only; the engine will apply deterministic information-value scoring after validation.
|
||||
- If the answer does not justify a change, return empty arrays for every category.
|
||||
- Use answerMeaning to preserve the answer's direct meaning even when the graph change remains unresolved.
|
||||
|
||||
## Example Constraint Reminder
|
||||
${formatExampleAnswerBlock()}
|
||||
|
||||
@@ -144,6 +144,29 @@ const graphUpdateNodeChangeSchema = z.object({
|
||||
reason: z.string().min(1),
|
||||
});
|
||||
|
||||
export const answerSupportCategory = /** @type {const} */ ({
|
||||
relative_priority_only: "relative_priority_only",
|
||||
conditional_tradeoff: "conditional_tradeoff",
|
||||
uncertain: "uncertain",
|
||||
explicit_hard_constraint: "explicit_hard_constraint",
|
||||
other: "other",
|
||||
});
|
||||
|
||||
export const answerResolutionGuidance = /** @type {const} */ ({
|
||||
must_remain_unresolved: "must_remain_unresolved",
|
||||
may_resolve: "may_resolve",
|
||||
must_resolve: "must_resolve",
|
||||
});
|
||||
|
||||
export const answerMeaningSchema = z
|
||||
.object({
|
||||
userSupportedMeaning: z.string().min(1),
|
||||
possibleInference: z.string().nullable().optional(),
|
||||
supportCategory: z.enum(Object.values(answerSupportCategory)),
|
||||
resolutionGuidance: z.enum(Object.values(answerResolutionGuidance)),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const selectedQuestionSchema = z
|
||||
.object({
|
||||
nodeId: z.string().min(1),
|
||||
@@ -160,6 +183,7 @@ export const graphUpdateSchema = z.object({
|
||||
resolvedUnknownNodeIds: z.array(z.string()).default([]),
|
||||
affectedNodeIds: z.array(z.string()).default([]),
|
||||
selectedQuestion: selectedQuestionSchema.nullable().default(null),
|
||||
answerMeaning: answerMeaningSchema.nullable().default(null),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof graphUpdateSchema>} GraphUpdate */
|
||||
|
||||
Reference in New Issue
Block a user