reasoning: stop answer fidelity guard blocking valid unclassified answers
This commit is contained in:
@@ -2689,6 +2689,70 @@ function normaliseSemanticText(value) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function semanticContentTokens(value) {
|
||||
const stopWords = new Set([
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"that",
|
||||
"this",
|
||||
"with",
|
||||
"from",
|
||||
"into",
|
||||
"than",
|
||||
"then",
|
||||
"they",
|
||||
"them",
|
||||
"their",
|
||||
"there",
|
||||
"about",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"because",
|
||||
"being",
|
||||
"been",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"were",
|
||||
"what",
|
||||
"when",
|
||||
"where",
|
||||
"which",
|
||||
"while",
|
||||
"mainly",
|
||||
"roughly",
|
||||
"specifically",
|
||||
"directly",
|
||||
"user",
|
||||
]);
|
||||
|
||||
return normaliseSemanticText(value)
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.split(" ")
|
||||
.filter((token) => token.length > 2 && !stopWords.has(token));
|
||||
}
|
||||
|
||||
function semanticOverlapRatio(sourceText, candidateText) {
|
||||
const source = new Set(semanticContentTokens(sourceText));
|
||||
const candidate = new Set(semanticContentTokens(candidateText));
|
||||
|
||||
if (candidate.size === 0) return 1;
|
||||
|
||||
const overlap = [...candidate].filter((token) => source.has(token)).length;
|
||||
return overlap / candidate.size;
|
||||
}
|
||||
|
||||
function rawAnswerSupportsUnclassifiedMeaning(answer, userSupportedMeaning) {
|
||||
const overlapRatio = semanticOverlapRatio(answer, userSupportedMeaning);
|
||||
const overlappingTokens = semanticContentTokens(userSupportedMeaning).filter(
|
||||
(token) => semanticContentTokens(answer).includes(token),
|
||||
).length;
|
||||
|
||||
return overlapRatio >= 0.4 || overlappingTokens >= 3;
|
||||
}
|
||||
|
||||
function hasConditionalQualification(text) {
|
||||
const value = normaliseSemanticText(text);
|
||||
return (
|
||||
@@ -2893,6 +2957,23 @@ function validateAnswerMeaningCompatibilityWithRawAnswer({ answer, proposal }) {
|
||||
}
|
||||
}
|
||||
|
||||
if (rawAnswerProfile.category === "other") {
|
||||
if (supportedMeaningProfile.category !== "other") {
|
||||
errors.push(
|
||||
"answerMeaning.userSupportedMeaning introduces a stronger reasoning category than the raw answer establishes.",
|
||||
);
|
||||
} else if (
|
||||
!rawAnswerSupportsUnclassifiedMeaning(
|
||||
answer,
|
||||
proposal.answerMeaning.userSupportedMeaning,
|
||||
)
|
||||
) {
|
||||
errors.push(
|
||||
"answerMeaning.userSupportedMeaning introduces unsupported meaning beyond what the raw answer itself states.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
@@ -2964,9 +3045,13 @@ function validateAnswerMeaningAlignment(proposal) {
|
||||
}
|
||||
|
||||
if (supportCategory === "other") {
|
||||
if (resolved || containsConstraintBoundaryLanguage(proposalText)) {
|
||||
if (
|
||||
resolved &&
|
||||
containsConstraintBoundaryLanguage(proposalText) &&
|
||||
!containsConstraintBoundaryLanguage(meaningText)
|
||||
) {
|
||||
errors.push(
|
||||
"Proposal cannot resolve or strengthen answerMeaning that does not clearly establish one of the protected reasoning categories.",
|
||||
"Proposal cannot resolve beyond an unclassified answer by introducing an unsupported constraint or preference/trade-off distinction.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1475,6 +1475,125 @@ describe("applyValidatedProposal", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("57A regression: a legitimate decision-advancing answer classified as other may resolve its intended unknown", () => {
|
||||
const graph = makeCommercialUpdateFixture();
|
||||
const parentId = graph.activeUnknownNodeId;
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer:
|
||||
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
|
||||
previousQuestion:
|
||||
"What problem would this need to solve to justify continuing development?",
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-other-people-problem",
|
||||
label: "Whether other people experience this problem",
|
||||
description:
|
||||
"Need to know whether other people experience this problem, because that must be established before deciding whether the problem is broadly important.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId,
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: parentId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"A concrete problem exists: reducing office overhead by roughly £2M annually is the main outcome being sought.",
|
||||
reason:
|
||||
"The answer directly states the practical decision-driving outcome the work is intended to achieve.",
|
||||
},
|
||||
],
|
||||
addedEdges: [
|
||||
makeEdge({
|
||||
id: "e-commercial-parent-other-people-problem",
|
||||
fromNodeId: parentId,
|
||||
toNodeId: "n-other-people-problem",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"After establishing the concrete problem in the current context, the next unknown is whether it also exists for other people.",
|
||||
}),
|
||||
],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [parentId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: {
|
||||
nodeId: "n-other-people-problem",
|
||||
question:
|
||||
"What makes you think other people experience this problem too?",
|
||||
reason:
|
||||
"The answer establishes the problem in this case; the next consequential unknown is whether it generalises beyond this case.",
|
||||
},
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"The main reason for considering this is cost reduction, specifically about £2M in annual office-overhead savings.",
|
||||
possibleInference:
|
||||
"If those savings are real and recurring, that could make the problem commercially important.",
|
||||
supportCategory: "other",
|
||||
resolutionGuidance: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(parentId);
|
||||
expect(result.stage).toBeUndefined();
|
||||
});
|
||||
|
||||
it("unsafe unclassified answer: other is not automatically trusted when the proposal adds stronger unsupported meaning", () => {
|
||||
const graph = makeCommercialUpdateFixture();
|
||||
const parentId = graph.activeUnknownNodeId;
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
answer:
|
||||
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
|
||||
previousQuestion:
|
||||
"What problem would this need to solve to justify continuing development?",
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: parentId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"There is proven broad market demand for a product that delivers these savings.",
|
||||
reason:
|
||||
"The answer was treated as establishing commercial demand rather than only the user's own cost-reduction goal.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [parentId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning:
|
||||
"There is proven broad market demand for a product that delivers these savings.",
|
||||
possibleInference:
|
||||
"The savings target could imply broader applicability if others share similar overhead pressures.",
|
||||
supportCategory: "other",
|
||||
resolutionGuidance: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors.join(" ")).toContain(
|
||||
"unsupported meaning beyond what the raw answer itself states",
|
||||
);
|
||||
});
|
||||
|
||||
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
|
||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||
|
||||
@@ -1690,7 +1809,7 @@ describe("applyValidatedProposal", () => {
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors.join(" ")).toContain(
|
||||
"does not clearly establish one of the protected reasoning categories",
|
||||
"unsupported constraint or preference/trade-off distinction",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user