feat(reasoning): add decision materiality rule
This commit is contained in:
@@ -134,6 +134,14 @@ The JSON object must contain exactly these top-level fields:
|
|||||||
31. If the answer explicitly states a hard constraint, state that directly in userSupportedMeaning.
|
31. If the answer explicitly states a hard constraint, state that directly in userSupportedMeaning.
|
||||||
32. Populate resolutionGuidance when the user's meaning genuinely implies must_remain_unresolved, may_resolve, or must_resolve. Keep it null only when no existing resolution state actually applies.
|
32. Populate resolutionGuidance when the user's meaning genuinely implies must_remain_unresolved, may_resolve, or must_resolve. Keep it null only when no existing resolution state actually applies.
|
||||||
|
|
||||||
|
## Decision Sufficiency Rule
|
||||||
|
|
||||||
|
An unresolved decision between options should not remain open merely because some uncertainty still exists.
|
||||||
|
|
||||||
|
Keep a decision context unresolved only when you can identify a specific unresolved factor that could materially change which option is preferred.
|
||||||
|
|
||||||
|
If the currently supported evidence is sufficient to distinguish the options and no such material unresolved factor remains, resolve the existing decision context and do not ask a generic continuation question.
|
||||||
|
|
||||||
## Decision Option Structure Rules
|
## Decision Option Structure Rules
|
||||||
When the user presents mutually exclusive candidate actions for one unresolved choice:
|
When the user presents mutually exclusive candidate actions for one unresolved choice:
|
||||||
|
|
||||||
|
|||||||
@@ -764,3 +764,102 @@ describe("60A.3 decision option prompt rules", () => {
|
|||||||
expect(altLine).toBeDefined();
|
expect(altLine).toBeDefined();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── Experiment 60B.4 — Decision Materiality Rule ──────────────
|
||||||
|
|
||||||
|
describe("60B.4 decision materiality rule", () => {
|
||||||
|
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||||
|
|
||||||
|
it("rule: unresolved decision not kept open merely because uncertainty remains", () => {
|
||||||
|
expect(prompt).toContain(
|
||||||
|
"should not remain open merely because some uncertainty still exists",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rule: continuation requires a specific unresolved factor", () => {
|
||||||
|
expect(prompt).toContain("specific unresolved factor");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rule: that factor must be capable of materially changing the preferred option", () => {
|
||||||
|
expect(prompt).toContain(
|
||||||
|
"could materially change which option is preferred",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rule: if no material factor remains, resolve existing decision context rather than asking generic question", () => {
|
||||||
|
expect(prompt).toContain(
|
||||||
|
"resolve the existing decision context",
|
||||||
|
);
|
||||||
|
expect(prompt).toContain(
|
||||||
|
"do not ask a generic continuation question",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT introduce financial thresholds", () => {
|
||||||
|
expect(prompt).not.toContain("threshold");
|
||||||
|
expect(prompt).not.toContain("£600k");
|
||||||
|
expect(prompt).not.toContain("£2m");
|
||||||
|
expect(prompt).not.toContain("payback");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT introduce relocation-specific examples", () => {
|
||||||
|
expect(prompt).not.toContain("relocation");
|
||||||
|
expect(prompt).not.toContain("Manchester");
|
||||||
|
expect(prompt).not.toContain("London");
|
||||||
|
expect(prompt).not.toContain("engineer");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT introduce automatic resolution whenever one option appears better", () => {
|
||||||
|
// The rule must not say "always resolve" or "resolve when one looks better"
|
||||||
|
const sufficiencySection = prompt.split(
|
||||||
|
"## Decision Sufficiency Rule",
|
||||||
|
)[1].split("## Decision Option Structure Rules")[0];
|
||||||
|
expect(sufficiencySection).not.toContain("always resolve");
|
||||||
|
expect(sufficiencySection).not.toContain(
|
||||||
|
"resolve when one option",
|
||||||
|
);
|
||||||
|
expect(sufficiencySection).not.toContain("looks better");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT introduce new schema fields or node kinds", () => {
|
||||||
|
// The rule text must not reference schema fields or node kinds outside the contract
|
||||||
|
expect(prompt).not.toContain("materiality_score");
|
||||||
|
expect(prompt).not.toContain("decision_sufficiency");
|
||||||
|
expect(prompt).not.toContain('kind "decision"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT mention specific currency or amounts", () => {
|
||||||
|
expect(prompt).not.toContain("£2 million");
|
||||||
|
expect(prompt).not.toContain("£600k");
|
||||||
|
expect(prompt).not.toContain("$1M");
|
||||||
|
expect(prompt).not.toContain("financial threshold");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("existing Rule 20 preserved", () => {
|
||||||
|
expect(prompt).toContain(
|
||||||
|
"Return selectedQuestion as null only when no consequential unresolved unknown remains.",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is domain-general — no provider-specific, taxonomic, or keyword-based language", () => {
|
||||||
|
expect(prompt).not.toContain("qwen");
|
||||||
|
expect(prompt).not.toContain("claude");
|
||||||
|
expect(prompt).not.toContain("gpt");
|
||||||
|
expect(prompt).not.toContain("keyword");
|
||||||
|
expect(prompt).not.toContain("synonym");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves the possibility of keeping a decision open when grounded factor exists", () => {
|
||||||
|
// The rule must allow continuation when a specific unresolved factor could change the outcome
|
||||||
|
expect(prompt).toContain("specific unresolved factor");
|
||||||
|
expect(prompt).toContain("could materially change which option is preferred");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("distinguishes uncertainty from decision-relevant uncertainty", () => {
|
||||||
|
// The rule must distinguish mere uncertainty from uncertainty that matters to the decision
|
||||||
|
const sufficiencySection = prompt.split(
|
||||||
|
"## Decision Sufficiency Rule",
|
||||||
|
)[1].split("## Decision Option Structure Rules")[0];
|
||||||
|
expect(sufficiencySection).toContain("should not remain open merely because some uncertainty still exists");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user