reasoning: stop answer fidelity guard blocking valid unclassified answers
This commit is contained in:
@@ -2689,6 +2689,70 @@ function normaliseSemanticText(value) {
|
|||||||
.trim();
|
.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function semanticContentTokens(value) {
|
||||||
|
const stopWords = new Set([
|
||||||
|
"the",
|
||||||
|
"and",
|
||||||
|
"for",
|
||||||
|
"that",
|
||||||
|
"this",
|
||||||
|
"with",
|
||||||
|
"from",
|
||||||
|
"into",
|
||||||
|
"than",
|
||||||
|
"then",
|
||||||
|
"they",
|
||||||
|
"them",
|
||||||
|
"their",
|
||||||
|
"there",
|
||||||
|
"about",
|
||||||
|
"would",
|
||||||
|
"could",
|
||||||
|
"should",
|
||||||
|
"because",
|
||||||
|
"being",
|
||||||
|
"been",
|
||||||
|
"have",
|
||||||
|
"has",
|
||||||
|
"had",
|
||||||
|
"were",
|
||||||
|
"what",
|
||||||
|
"when",
|
||||||
|
"where",
|
||||||
|
"which",
|
||||||
|
"while",
|
||||||
|
"mainly",
|
||||||
|
"roughly",
|
||||||
|
"specifically",
|
||||||
|
"directly",
|
||||||
|
"user",
|
||||||
|
]);
|
||||||
|
|
||||||
|
return normaliseSemanticText(value)
|
||||||
|
.replace(/[^a-z0-9]+/g, " ")
|
||||||
|
.split(" ")
|
||||||
|
.filter((token) => token.length > 2 && !stopWords.has(token));
|
||||||
|
}
|
||||||
|
|
||||||
|
function semanticOverlapRatio(sourceText, candidateText) {
|
||||||
|
const source = new Set(semanticContentTokens(sourceText));
|
||||||
|
const candidate = new Set(semanticContentTokens(candidateText));
|
||||||
|
|
||||||
|
if (candidate.size === 0) return 1;
|
||||||
|
|
||||||
|
const overlap = [...candidate].filter((token) => source.has(token)).length;
|
||||||
|
return overlap / candidate.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rawAnswerSupportsUnclassifiedMeaning(answer, userSupportedMeaning) {
|
||||||
|
const overlapRatio = semanticOverlapRatio(answer, userSupportedMeaning);
|
||||||
|
const overlappingTokens = semanticContentTokens(userSupportedMeaning).filter(
|
||||||
|
(token) => semanticContentTokens(answer).includes(token),
|
||||||
|
).length;
|
||||||
|
|
||||||
|
return overlapRatio >= 0.4 || overlappingTokens >= 3;
|
||||||
|
}
|
||||||
|
|
||||||
function hasConditionalQualification(text) {
|
function hasConditionalQualification(text) {
|
||||||
const value = normaliseSemanticText(text);
|
const value = normaliseSemanticText(text);
|
||||||
return (
|
return (
|
||||||
@@ -2893,6 +2957,23 @@ function validateAnswerMeaningCompatibilityWithRawAnswer({ answer, proposal }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (rawAnswerProfile.category === "other") {
|
||||||
|
if (supportedMeaningProfile.category !== "other") {
|
||||||
|
errors.push(
|
||||||
|
"answerMeaning.userSupportedMeaning introduces a stronger reasoning category than the raw answer establishes.",
|
||||||
|
);
|
||||||
|
} else if (
|
||||||
|
!rawAnswerSupportsUnclassifiedMeaning(
|
||||||
|
answer,
|
||||||
|
proposal.answerMeaning.userSupportedMeaning,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
errors.push(
|
||||||
|
"answerMeaning.userSupportedMeaning introduces unsupported meaning beyond what the raw answer itself states.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return errors;
|
return errors;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2964,9 +3045,13 @@ function validateAnswerMeaningAlignment(proposal) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (supportCategory === "other") {
|
if (supportCategory === "other") {
|
||||||
if (resolved || containsConstraintBoundaryLanguage(proposalText)) {
|
if (
|
||||||
|
resolved &&
|
||||||
|
containsConstraintBoundaryLanguage(proposalText) &&
|
||||||
|
!containsConstraintBoundaryLanguage(meaningText)
|
||||||
|
) {
|
||||||
errors.push(
|
errors.push(
|
||||||
"Proposal cannot resolve or strengthen answerMeaning that does not clearly establish one of the protected reasoning categories.",
|
"Proposal cannot resolve beyond an unclassified answer by introducing an unsupported constraint or preference/trade-off distinction.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1475,6 +1475,125 @@ describe("applyValidatedProposal", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("57A regression: a legitimate decision-advancing answer classified as other may resolve its intended unknown", () => {
|
||||||
|
const graph = makeCommercialUpdateFixture();
|
||||||
|
const parentId = graph.activeUnknownNodeId;
|
||||||
|
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: graph,
|
||||||
|
answer:
|
||||||
|
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
|
||||||
|
previousQuestion:
|
||||||
|
"What problem would this need to solve to justify continuing development?",
|
||||||
|
proposal: {
|
||||||
|
addedNodes: [
|
||||||
|
makeNode({
|
||||||
|
id: "n-other-people-problem",
|
||||||
|
label: "Whether other people experience this problem",
|
||||||
|
description:
|
||||||
|
"Need to know whether other people experience this problem, because that must be established before deciding whether the problem is broadly important.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
parentId,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
updatedNodes: [
|
||||||
|
{
|
||||||
|
nodeId: parentId,
|
||||||
|
previousStatus: "unknown",
|
||||||
|
newStatus: "resolved",
|
||||||
|
previousValue: null,
|
||||||
|
newValue:
|
||||||
|
"A concrete problem exists: reducing office overhead by roughly £2M annually is the main outcome being sought.",
|
||||||
|
reason:
|
||||||
|
"The answer directly states the practical decision-driving outcome the work is intended to achieve.",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
addedEdges: [
|
||||||
|
makeEdge({
|
||||||
|
id: "e-commercial-parent-other-people-problem",
|
||||||
|
fromNodeId: parentId,
|
||||||
|
toNodeId: "n-other-people-problem",
|
||||||
|
relationship: "depends_on",
|
||||||
|
confidence: "medium",
|
||||||
|
description:
|
||||||
|
"After establishing the concrete problem in the current context, the next unknown is whether it also exists for other people.",
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
removedEdgeIds: [],
|
||||||
|
resolvedUnknownNodeIds: [parentId],
|
||||||
|
affectedNodeIds: [],
|
||||||
|
selectedQuestion: {
|
||||||
|
nodeId: "n-other-people-problem",
|
||||||
|
question:
|
||||||
|
"What makes you think other people experience this problem too?",
|
||||||
|
reason:
|
||||||
|
"The answer establishes the problem in this case; the next consequential unknown is whether it generalises beyond this case.",
|
||||||
|
},
|
||||||
|
answerMeaning: {
|
||||||
|
userSupportedMeaning:
|
||||||
|
"The main reason for considering this is cost reduction, specifically about £2M in annual office-overhead savings.",
|
||||||
|
possibleInference:
|
||||||
|
"If those savings are real and recurring, that could make the problem commercially important.",
|
||||||
|
supportCategory: "other",
|
||||||
|
resolutionGuidance: null,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(parentId);
|
||||||
|
expect(result.stage).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("unsafe unclassified answer: other is not automatically trusted when the proposal adds stronger unsupported meaning", () => {
|
||||||
|
const graph = makeCommercialUpdateFixture();
|
||||||
|
const parentId = graph.activeUnknownNodeId;
|
||||||
|
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: graph,
|
||||||
|
answer:
|
||||||
|
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
|
||||||
|
previousQuestion:
|
||||||
|
"What problem would this need to solve to justify continuing development?",
|
||||||
|
proposal: {
|
||||||
|
addedNodes: [],
|
||||||
|
updatedNodes: [
|
||||||
|
{
|
||||||
|
nodeId: parentId,
|
||||||
|
previousStatus: "unknown",
|
||||||
|
newStatus: "resolved",
|
||||||
|
previousValue: null,
|
||||||
|
newValue:
|
||||||
|
"There is proven broad market demand for a product that delivers these savings.",
|
||||||
|
reason:
|
||||||
|
"The answer was treated as establishing commercial demand rather than only the user's own cost-reduction goal.",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
addedEdges: [],
|
||||||
|
removedEdgeIds: [],
|
||||||
|
resolvedUnknownNodeIds: [parentId],
|
||||||
|
affectedNodeIds: [],
|
||||||
|
selectedQuestion: null,
|
||||||
|
answerMeaning: {
|
||||||
|
userSupportedMeaning:
|
||||||
|
"There is proven broad market demand for a product that delivers these savings.",
|
||||||
|
possibleInference:
|
||||||
|
"The savings target could imply broader applicability if others share similar overhead pressures.",
|
||||||
|
supportCategory: "other",
|
||||||
|
resolutionGuidance: null,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
expect(result.stage).toBe("proposal_compatibility");
|
||||||
|
expect(result.errors.join(" ")).toContain(
|
||||||
|
"unsupported meaning beyond what the raw answer itself states",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
|
it("Regression C: rejects unresolved uncertainty being treated as resolved", () => {
|
||||||
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
const { graph, riskUnknownId } = makeRiskClarificationFixture();
|
||||||
|
|
||||||
@@ -1690,7 +1809,7 @@ describe("applyValidatedProposal", () => {
|
|||||||
expect(result.success).toBe(false);
|
expect(result.success).toBe(false);
|
||||||
expect(result.stage).toBe("proposal_compatibility");
|
expect(result.stage).toBe("proposal_compatibility");
|
||||||
expect(result.errors.join(" ")).toContain(
|
expect(result.errors.join(" ")).toContain(
|
||||||
"does not clearly establish one of the protected reasoning categories",
|
"unsupported constraint or preference/trade-off distinction",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user