reasoning: stop answer fidelity guard blocking valid unclassified answers

This commit is contained in:
2026-08-09 19:55:35 +01:00
parent 7d94c6f73a
commit 4e4d0fa732
2 changed files with 207 additions and 3 deletions
+87 -2
View File
@@ -2689,6 +2689,70 @@ function normaliseSemanticText(value) {
.trim();
}
function semanticContentTokens(value) {
const stopWords = new Set([
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"into",
"than",
"then",
"they",
"them",
"their",
"there",
"about",
"would",
"could",
"should",
"because",
"being",
"been",
"have",
"has",
"had",
"were",
"what",
"when",
"where",
"which",
"while",
"mainly",
"roughly",
"specifically",
"directly",
"user",
]);
return normaliseSemanticText(value)
.replace(/[^a-z0-9]+/g, " ")
.split(" ")
.filter((token) => token.length > 2 && !stopWords.has(token));
}
function semanticOverlapRatio(sourceText, candidateText) {
const source = new Set(semanticContentTokens(sourceText));
const candidate = new Set(semanticContentTokens(candidateText));
if (candidate.size === 0) return 1;
const overlap = [...candidate].filter((token) => source.has(token)).length;
return overlap / candidate.size;
}
function rawAnswerSupportsUnclassifiedMeaning(answer, userSupportedMeaning) {
const overlapRatio = semanticOverlapRatio(answer, userSupportedMeaning);
const overlappingTokens = semanticContentTokens(userSupportedMeaning).filter(
(token) => semanticContentTokens(answer).includes(token),
).length;
return overlapRatio >= 0.4 || overlappingTokens >= 3;
}
function hasConditionalQualification(text) {
const value = normaliseSemanticText(text);
return (
@@ -2893,6 +2957,23 @@ function validateAnswerMeaningCompatibilityWithRawAnswer({ answer, proposal }) {
}
}
if (rawAnswerProfile.category === "other") {
if (supportedMeaningProfile.category !== "other") {
errors.push(
"answerMeaning.userSupportedMeaning introduces a stronger reasoning category than the raw answer establishes.",
);
} else if (
!rawAnswerSupportsUnclassifiedMeaning(
answer,
proposal.answerMeaning.userSupportedMeaning,
)
) {
errors.push(
"answerMeaning.userSupportedMeaning introduces unsupported meaning beyond what the raw answer itself states.",
);
}
}
return errors;
}
@@ -2964,9 +3045,13 @@ function validateAnswerMeaningAlignment(proposal) {
}
if (supportCategory === "other") {
if (resolved || containsConstraintBoundaryLanguage(proposalText)) {
if (
resolved &&
containsConstraintBoundaryLanguage(proposalText) &&
!containsConstraintBoundaryLanguage(meaningText)
) {
errors.push(
"Proposal cannot resolve or strengthen answerMeaning that does not clearly establish one of the protected reasoning categories.",
"Proposal cannot resolve beyond an unclassified answer by introducing an unsupported constraint or preference/trade-off distinction.",
);
}
}
+120 -1
View File
@@ -1475,6 +1475,125 @@ describe("applyValidatedProposal", () => {
);
});
it("57A regression: a legitimate decision-advancing answer classified as other may resolve its intended unknown", () => {
const graph = makeCommercialUpdateFixture();
const parentId = graph.activeUnknownNodeId;
const result = applyValidatedProposal({
situationGraph: graph,
answer:
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
previousQuestion:
"What problem would this need to solve to justify continuing development?",
proposal: {
addedNodes: [
makeNode({
id: "n-other-people-problem",
label: "Whether other people experience this problem",
description:
"Need to know whether other people experience this problem, because that must be established before deciding whether the problem is broadly important.",
kind: "unknown",
status: "unknown",
confidence: "medium",
parentId,
}),
],
updatedNodes: [
{
nodeId: parentId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"A concrete problem exists: reducing office overhead by roughly £2M annually is the main outcome being sought.",
reason:
"The answer directly states the practical decision-driving outcome the work is intended to achieve.",
},
],
addedEdges: [
makeEdge({
id: "e-commercial-parent-other-people-problem",
fromNodeId: parentId,
toNodeId: "n-other-people-problem",
relationship: "depends_on",
confidence: "medium",
description:
"After establishing the concrete problem in the current context, the next unknown is whether it also exists for other people.",
}),
],
removedEdgeIds: [],
resolvedUnknownNodeIds: [parentId],
affectedNodeIds: [],
selectedQuestion: {
nodeId: "n-other-people-problem",
question:
"What makes you think other people experience this problem too?",
reason:
"The answer establishes the problem in this case; the next consequential unknown is whether it generalises beyond this case.",
},
answerMeaning: {
userSupportedMeaning:
"The main reason for considering this is cost reduction, specifically about £2M in annual office-overhead savings.",
possibleInference:
"If those savings are real and recurring, that could make the problem commercially important.",
supportCategory: "other",
resolutionGuidance: null,
},
},
});
expect(result.success).toBe(true);
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(parentId);
expect(result.stage).toBeUndefined();
});
it("unsafe unclassified answer: other is not automatically trusted when the proposal adds stronger unsupported meaning", () => {
const graph = makeCommercialUpdateFixture();
const parentId = graph.activeUnknownNodeId;
const result = applyValidatedProposal({
situationGraph: graph,
answer:
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
previousQuestion:
"What problem would this need to solve to justify continuing development?",
proposal: {
addedNodes: [],
updatedNodes: [
{
nodeId: parentId,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue:
"There is proven broad market demand for a product that delivers these savings.",
reason:
"The answer was treated as establishing commercial demand rather than only the user's own cost-reduction goal.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [parentId],
affectedNodeIds: [],
selectedQuestion: null,
answerMeaning: {
userSupportedMeaning:
"There is proven broad market demand for a product that delivers these savings.",
possibleInference:
"The savings target could imply broader applicability if others share similar overhead pressures.",
supportCategory: "other",
resolutionGuidance: null,
},
},
});
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain(
"unsupported meaning beyond what the raw answer itself states",
);
});
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
const { graph, riskUnknownId } = makeRiskClarificationFixture();
@@ -1690,7 +1809,7 @@ describe("applyValidatedProposal", () => {
expect(result.success).toBe(false);
expect(result.stage).toBe("proposal_compatibility");
expect(result.errors.join(" ")).toContain(
"does not clearly establish one of the protected reasoning categories",
"unsupported constraint or preference/trade-off distinction",
);
});