test(experiment): checkpoint three-turn separated reasoning apparatus
This commit is contained in:
@@ -0,0 +1,456 @@
|
||||
/**
|
||||
* RTO.17A — Three-turn separated reasoning stability apparatus.
|
||||
*
|
||||
* Purpose: Test whether one model invocation can keep focused investigation
|
||||
* understanding and wider decision-significance reasoning explicitly separate
|
||||
* after a THIRD user-chosen turn, using the RTO.16 live result as prior state.
|
||||
*
|
||||
* Design boundary:
|
||||
* - Standalone experimental runner.
|
||||
* - Zero production code changes.
|
||||
* - Inspect-only by default (zero live model calls).
|
||||
* - --live flag enables exactly one live call.
|
||||
*
|
||||
* Output contract for each layer (reused from RTO.16):
|
||||
* focusedUnderstanding: { targetNodeId, observations, uncertainties,
|
||||
* assumptions, relationships, possibleFollowUpQuestions }
|
||||
* decisionSignificance: [{ insight }] (concise wider-decision connections)
|
||||
*/
|
||||
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// ─── Fixed inputs — exact RTO.15 competitor/IP scenario ──────────────────────
|
||||
|
||||
const TARGET_NODE_ID = "nxmeiab";
|
||||
|
||||
const CENTRAL_CASE_STATEMENT =
|
||||
"I am deciding whether to launch a new software product this year or wait twelve months. The product is ready enough to launch, but one large enterprise customer could represent a significant part of the expected revenue and I do not yet know whether they will sign. Launching this year would also require around £300,000 of additional support and implementation cost. Waiting twelve months would reduce that immediate cost and give us more time to improve the product, but it would delay revenue and may allow competitors to move first. I need to decide whether there is enough evidence to launch this year or whether waiting is the safer decision.";
|
||||
|
||||
const TARGET_LABEL =
|
||||
"Whether competitors are actively developing similar products and how soon they might release them";
|
||||
|
||||
// ─── RTO.16 prior state — loaded from live artifact as clean state boundary ──
|
||||
|
||||
async function loadPriorState() {
|
||||
const resultsDir = path.resolve("tests/experimental/results");
|
||||
const artifactPath = path.resolve(resultsDir, "rto-layer-separated-live.json");
|
||||
|
||||
try {
|
||||
const raw = await fs.readFile(artifactPath, "utf8");
|
||||
const artifact = JSON.parse(raw);
|
||||
return artifact.structuredResult;
|
||||
} catch (err) {
|
||||
if (err.code === "ENOENT") {
|
||||
throw new Error("RTO.16 live artifact not found at " + artifactPath);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Third-turn fixed data ──────────────────────────────────────────────────
|
||||
|
||||
const TURN_3_FOLLOW_UP =
|
||||
"Can the content of the competitor's conference presentation be analyzed to distinguish between technical R&D and general market education?";
|
||||
|
||||
const TURN_3_ANSWER =
|
||||
"The conference presentation included a technical architecture diagram,\n" +
|
||||
"a prototype workflow and discussion of model-training challenges that\n" +
|
||||
"closely match the customer problem we are solving. It did not name a\n" +
|
||||
"commercial product or launch date, but it appears more consistent with\n" +
|
||||
"active product development than general market education. I still do\n" +
|
||||
"not know whether the prototype can match our patented processes or\n" +
|
||||
"whether it is intended for the same enterprise customers.";
|
||||
|
||||
// ─── Reasoning instructions — explicit layer separation (RTO.16 contract) ───
|
||||
|
||||
const REASONING_INSTRUCTIONS = `You are performing a THIRD-TURN focused-investigation refinement for one user-chosen investigation node.
|
||||
|
||||
A previous turn produced local observations, uncertainties, assumptions and relationships about this node. Those findings have NOT been merged into any global graph — they exist only as a compact local investigation state.
|
||||
|
||||
The user has now answered one follow-up question from that prior state. Your task is to produce ONE updated response with TWO EXPLICITLY SEPARATE LAYERS.
|
||||
|
||||
## Layer 1: Focused Investigation Understanding
|
||||
|
||||
Maintain one current focused investigation understanding for the chosen node. Ground it only in:
|
||||
- The prior accumulated focused state (turns 1–2)
|
||||
- The user-chosen follow-up question (turn 3)
|
||||
- The new answer to that follow-up (turn 3)
|
||||
|
||||
Within this layer, your observations and uncertainties must reflect ONLY what is directly observable from the combined evidence about this investigation node. Keep prior information that remains supported. Revise or remove information that is no longer accurate.
|
||||
|
||||
## Layer 2: Wider Decision Significance
|
||||
|
||||
Separately identify concise connections explaining why the current focused understanding matters to the wider decision. Use the central case statement provided below for this layer only.
|
||||
|
||||
Each significance insight should be brief — one sentence connecting a focused finding to a broader business implication (timing, cost, competitive risk, strategic optionality).
|
||||
|
||||
## Critical separation rules
|
||||
|
||||
1. Layer 1 MUST NOT absorb wider-case concerns merely because they are present in the central case statement.
|
||||
2. Observations must only state what the combined evidence directly supports about this investigation node — no speculative decision implications.
|
||||
3. Uncertainties must distinguish between investigative gaps (Layer 1) and decision uncertainty (Layer 2).
|
||||
4. Assumptions surviving from prior turns stay in Layer 1; wider-case financial or strategic constraints appear only in Layer 2.
|
||||
5. Layer 2 insights must be concise — not another full investigation report.
|
||||
6. Do not duplicate facts between layers unnecessarily.
|
||||
7. Do NOT recommend what the user should do.
|
||||
8. Do NOT return turn-by-turn history.
|
||||
9. Treat the supplied focusedUnderstanding and decisionSignificance as the current state.
|
||||
10. Return their revised current forms after considering the new question and answer.
|
||||
|
||||
Return exactly one JSON object with two top-level fields:
|
||||
- focusedUnderstanding (object with targetNodeId, observations, uncertainties, assumptions, relationships, possibleFollowUpQuestions)
|
||||
- decisionSignificance (array of objects with "insight" field)`;
|
||||
|
||||
// ─── Prompt construction ─────────────────────────────────────────────────────
|
||||
|
||||
function buildPrompt({ priorState, includeCentralCase = false }) {
|
||||
const centralSection = includeCentralCase
|
||||
? "\nCentral case statement:\n" + CENTRAL_CASE_STATEMENT + "\n"
|
||||
: "";
|
||||
|
||||
return (
|
||||
REASONING_INSTRUCTIONS +
|
||||
|
||||
"\n\n## Prior accumulated focused state (turns 1–2)\n" +
|
||||
|
||||
"### Observations\n" +
|
||||
priorState.focusedUnderstanding.observations.map((o) => "- " + o).join("\n") +
|
||||
|
||||
"\n\n### Uncertainties\n" +
|
||||
priorState.focusedUnderstanding.uncertainties.map((u) => "- " + u).join("\n") +
|
||||
|
||||
"\n\n### Assumptions\n" +
|
||||
priorState.focusedUnderstanding.assumptions.map((a) => "- " + a).join("\n") +
|
||||
|
||||
"\n\n### Relationships\n" +
|
||||
priorState.focusedUnderstanding.relationships.map((r) => "- " + r.from + " → " + r.to + " (" + r.type + ")").join("\n") +
|
||||
|
||||
"\n\n### Possible follow-up questions (from prior state)\n" +
|
||||
priorState.focusedUnderstanding.possibleFollowUpQuestions.map((q) => "- " + q).join("\n") +
|
||||
|
||||
"\n\n## Current decision significance (wider case implications)\n" +
|
||||
priorState.decisionSignificance.map((s) => "- " + s.insight).join("\n") +
|
||||
|
||||
"\n\n## This turn's input\n" +
|
||||
|
||||
"Target label: " + TARGET_LABEL + centralSection +
|
||||
'Previous follow-up question: "Can the content of the competitor\'s conference presentation be analyzed to distinguish between technical R&D and general market education?"\n' +
|
||||
|
||||
"User-chosen follow-up question (turn 3):\n" +
|
||||
TURN_3_FOLLOW_UP +
|
||||
|
||||
"\n\nAnswer to follow-up question (turn 3):\n" +
|
||||
TURN_3_ANSWER +
|
||||
|
||||
"\n\n## Output format\n" +
|
||||
|
||||
"Return exactly one JSON object with these two top-level fields:\n" +
|
||||
|
||||
"### focusedUnderstanding (object)\n" +
|
||||
'- targetNodeId: "' + TARGET_NODE_ID + '"\n' +
|
||||
"- observations: array of strings — what the combined evidence supports as factual, scoped to this investigation node only\n" +
|
||||
"- uncertainties: array of strings — what remains unknown or unclear after all answers, about this investigation node\n" +
|
||||
"- assumptions: array of strings — supporting beliefs that persist, revised if needed\n" +
|
||||
"- relationships: array of { from, to, type } — links within the investigation\n" +
|
||||
"- possibleFollowUpQuestions: array of strings — genuinely remaining new questions\n" +
|
||||
|
||||
"\n### decisionSignificance (array)\n" +
|
||||
'- Each item: { insight: string } — a concise connection between the current focused understanding and wider-case implications\n' +
|
||||
|
||||
"\nDo NOT mix wider-case concerns into focusedUnderstanding. Do NOT recommend a decision. Return one coherent response with two clearly separated purposes."
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Schema for live mode result validation (reused from RTO.16 contract) ────
|
||||
|
||||
const RESULT_SCHEMA = {
|
||||
type: "object",
|
||||
required: ["focusedUnderstanding", "decisionSignificance"],
|
||||
properties: {
|
||||
focusedUnderstanding: {
|
||||
type: "object",
|
||||
required: [
|
||||
"targetNodeId",
|
||||
"observations",
|
||||
"uncertainties",
|
||||
"assumptions",
|
||||
"relationships",
|
||||
"possibleFollowUpQuestions",
|
||||
],
|
||||
},
|
||||
decisionSignificance: {
|
||||
type: "array",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// ─── Inspect-only mode (default: zero model calls) ───────────────────────────
|
||||
|
||||
async function inspectApparatus() {
|
||||
// Validate prior artifact exists and is usable
|
||||
let priorState;
|
||||
try {
|
||||
priorState = await loadPriorState();
|
||||
} catch (err) {
|
||||
console.error("=== RTO.17A Three-Turn Apparatus (inspect-only) ===\n");
|
||||
console.error("FAIL: " + err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Verify target identity
|
||||
const targetMatch = priorState.focusedUnderstanding.targetNodeId === TARGET_NODE_ID;
|
||||
|
||||
// Verify third-turn question comes from prior possibleFollowUpQuestions
|
||||
const questionExistsInPrior =
|
||||
priorState.focusedUnderstanding.possibleFollowUpQuestions.includes(TURN_3_FOLLOW_UP);
|
||||
|
||||
// Build prompts for inspection
|
||||
const localPrompt = buildPrompt({ priorState, includeCentralCase: false });
|
||||
const caseAwarePrompt = buildPrompt({ priorState, includeCentralCase: true });
|
||||
|
||||
// Structural validation for both layers
|
||||
function validatePrompt(prompt, label) {
|
||||
var requiredSections = [
|
||||
"Prior accumulated focused state (turns 1–2)",
|
||||
"### Observations",
|
||||
"### Uncertainties",
|
||||
"### Assumptions",
|
||||
"### Relationships",
|
||||
"follow-up questions (from prior state)",
|
||||
"Current decision significance",
|
||||
"User-chosen follow-up question (turn 3)",
|
||||
"Answer to follow-up question (turn 3)",
|
||||
"Treat the supplied focusedUnderstanding and decisionSignificance as the current state.",
|
||||
"Return exactly one JSON object",
|
||||
"focusedUnderstanding",
|
||||
"decisionSignificance",
|
||||
"Do NOT recommend a decision",
|
||||
"Do NOT return turn-by-turn history",
|
||||
];
|
||||
|
||||
var missing = requiredSections.filter(function (s) { return prompt.indexOf(s) === -1; });
|
||||
var present = requiredSections.filter(function (s) { return prompt.indexOf(s) !== -1; });
|
||||
|
||||
return {
|
||||
label: label,
|
||||
characterCount: prompt.length,
|
||||
tokenEstimate: Math.ceil(prompt.length / 4),
|
||||
sectionsRequired: requiredSections.length,
|
||||
sectionsPresent: present.length,
|
||||
missingSections: missing,
|
||||
isValid: missing.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
var localInspect = validatePrompt(localPrompt, "local-only");
|
||||
var caseAwareInspect = validatePrompt(caseAwarePrompt, "case-aware-with-central-statement");
|
||||
|
||||
// Verify layer separation instructions are present
|
||||
function validateSeparationRules(prompt) {
|
||||
var rules = [
|
||||
"MUST NOT absorb wider-case concerns merely because",
|
||||
"Layer 1 MUST NOT absorb",
|
||||
"Layer 2 insights must be concise",
|
||||
"Do NOT mix wider-case concerns",
|
||||
"Do NOT recommend a decision",
|
||||
"Do NOT return turn-by-turn history",
|
||||
"exactly one JSON object with two top-level fields",
|
||||
"Treat the supplied focusedUnderstanding and decisionSignificance as the current state",
|
||||
];
|
||||
var missingRules = rules.filter(function (r) { return prompt.indexOf(r) === -1; });
|
||||
return {
|
||||
rulesRequired: rules.length,
|
||||
rulesPresent: rules.length - missingRules.length,
|
||||
missingRules: missingRules,
|
||||
separationInstructionsComplete: missingRules.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
var localSeparation = validateSeparationRules(localPrompt);
|
||||
var caseAwareSeparation = validateSeparationRules(caseAwarePrompt);
|
||||
|
||||
function describeFocusedFields() {
|
||||
return [
|
||||
"targetNodeId (string)",
|
||||
"observations (array<string>)",
|
||||
"uncertainties (array<string>)",
|
||||
"assumptions (array<string>)",
|
||||
"relationships (array<{from, to, type}>)",
|
||||
"possibleFollowUpQuestions (array<string>)",
|
||||
];
|
||||
}
|
||||
|
||||
var inputCharCount = localPrompt.length;
|
||||
|
||||
console.log("=== RTO.17A Three-Turn Separated Reasoning Apparatus (inspect-only) ===\n");
|
||||
|
||||
// Report per task spec
|
||||
console.log("--- Prior state validation ---");
|
||||
console.log("Prior artifact exists: YES");
|
||||
console.log("Prior targetNodeId matches (nxmeiab): " + targetMatch);
|
||||
console.log("Third-turn question from prior possibleFollowUpQuestions: " + questionExistsInPrior);
|
||||
console.log("Prior observations count: " + priorState.focusedUnderstanding.observations.length);
|
||||
console.log("Prior uncertainties count: " + priorState.focusedUnderstanding.uncertainties.length);
|
||||
console.log("Prior assumptions count: " + priorState.focusedUnderstanding.assumptions.length);
|
||||
console.log("Prior possibleFollowUpQuestions count: " + priorState.focusedUnderstanding.possibleFollowUpQuestions.length);
|
||||
|
||||
console.log("\n--- Separated input ---");
|
||||
console.log("focusedUnderstanding fields: " + describeFocusedFields().join(", "));
|
||||
console.log("decisionSignificance shape: array<{insight: string}>");
|
||||
console.log("whole SituationGraph supplied: NO");
|
||||
console.log("global selector supplied: NO");
|
||||
console.log("turn history supplied: NO");
|
||||
console.log("live route available: YES (via --live flag)");
|
||||
|
||||
console.log("\n--- Prompt inspection ---");
|
||||
console.log(JSON.stringify({
|
||||
experiment: "rto-three-turn-separated-reasoning",
|
||||
artifactType: "APPARATUS INSPECTION",
|
||||
nonProductionApparatus: true,
|
||||
productionCodeUnchanged: true,
|
||||
inputCharacterCount: {
|
||||
localOnly: localInspect.characterCount,
|
||||
caseAware: caseAwareInspect.characterCount,
|
||||
},
|
||||
promptValidation: {
|
||||
localOnly: localInspect,
|
||||
caseAwareWithCentralStatement: caseAwareInspect,
|
||||
},
|
||||
layerSeparationInstructions: {
|
||||
localOnly: localSeparation,
|
||||
caseAwareWithCentralStatement: caseAwareSeparation,
|
||||
},
|
||||
priorStateFromRto16: {
|
||||
targetNodeId: priorState.focusedUnderstanding.targetNodeId,
|
||||
observationsCount: priorState.focusedUnderstanding.observations.length,
|
||||
uncertaintiesCount: priorState.focusedUnderstanding.uncertainties.length,
|
||||
assumptionsCount: priorState.focusedUnderstanding.assumptions.length,
|
||||
followUpQuestionsCount: priorState.focusedUnderstanding.possibleFollowUpQuestions.length,
|
||||
decisionSignificanceCount: priorState.decisionSignificance.length,
|
||||
},
|
||||
thirdTurnData: {
|
||||
followUpQuestion: TURN_3_FOLLOW_UP,
|
||||
answerPreservedExactly: true,
|
||||
questionSource: "prior possibleFollowUpQuestions[0]",
|
||||
},
|
||||
doesNotRequire: [
|
||||
"whole SituationGraph",
|
||||
"global selector",
|
||||
"production focused-investigation.js modifications",
|
||||
"production API routes",
|
||||
"UI changes",
|
||||
"SituationGraph updates",
|
||||
"global update path changes",
|
||||
"question formulation changes",
|
||||
"turn-by-turn transcript",
|
||||
],
|
||||
}, null, 2));
|
||||
|
||||
return { inputCharacterCount: inputCharCount };
|
||||
}
|
||||
|
||||
// ─── Live execution (single invocation) ──────────────────────────────────────
|
||||
|
||||
async function executeLiveMode() {
|
||||
var baseUrl = process.env.OLLAMA_BASE_URL;
|
||||
if (!baseUrl) {
|
||||
throw new Error("OLLAMA_BASE_URL not set. Cannot execute live mode.");
|
||||
}
|
||||
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||
throw new Error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
|
||||
}
|
||||
|
||||
// Import zod for live schema validation (avoid in inspect-only)
|
||||
var z = (await import("zod")).z;
|
||||
|
||||
var focusedSchema = z.object({
|
||||
targetNodeId: z.literal(TARGET_NODE_ID),
|
||||
observations: z.array(z.string()),
|
||||
uncertainties: z.array(z.string()),
|
||||
assumptions: z.array(z.string()),
|
||||
relationships: z.array(
|
||||
z.object({ from: z.string(), to: z.string(), type: z.string() }),
|
||||
),
|
||||
possibleFollowUpQuestions: z.array(z.string()),
|
||||
});
|
||||
|
||||
var resultSchema = z.object({
|
||||
focusedUnderstanding: focusedSchema,
|
||||
decisionSignificance: z.array(z.object({ insight: z.string() })),
|
||||
}).strict();
|
||||
|
||||
// Load prior state
|
||||
var priorState = await loadPriorState();
|
||||
|
||||
// Import provider dynamically
|
||||
var { getProvider } = await import(path.resolve(__dirname, "../../lib/llm/provider.js"));
|
||||
var provider = getProvider();
|
||||
|
||||
var configEnv = await import(path.resolve(__dirname, "../../lib/config.js"));
|
||||
var modelName = configEnv.assertConfig().OLLAMA_MODEL;
|
||||
if (!modelName) {
|
||||
throw new Error("OLLAMA_MODEL not set.");
|
||||
}
|
||||
|
||||
// Build the single prompt (case-aware: includes central case statement)
|
||||
var prompt = buildPrompt({ priorState, includeCentralCase: true });
|
||||
var startedAt = Date.now();
|
||||
console.log("Sending to " + provider.name + " (" + modelName + ")...");
|
||||
|
||||
var raw = await provider.generateReconstruction(prompt, modelName);
|
||||
var elapsedMs = Date.now() - startedAt;
|
||||
|
||||
var parsedResult = resultSchema.parse(raw);
|
||||
|
||||
// Write result artifact for later comparison against RTO.16
|
||||
var resultsDir = path.resolve("tests/experimental/results");
|
||||
await fs.mkdir(resultsDir, { recursive: true });
|
||||
var artifactPath = path.resolve(resultsDir, "rto-three-turn-separated-live.json");
|
||||
var payload = {
|
||||
apparatus: "rto-three-turn-separated-reasoning.mjs",
|
||||
experiment: "RTO.17A",
|
||||
artifactType: "LIVE RESULT — RTO.17A three-turn separated layers invocation",
|
||||
modelName: modelName,
|
||||
elapsedMs: elapsedMs,
|
||||
promptLength: prompt.length,
|
||||
inputCharacterCount: prompt.length,
|
||||
priorArtifactUsed: "rto-layer-separated-live.json (RTO.16)",
|
||||
priorFocusedUnderstandingLoaded: true,
|
||||
priorDecisionSignificanceLoaded: true,
|
||||
centralCaseStatementSupplied: true,
|
||||
turnHistorySupplied: false,
|
||||
layerStructure: { focusedUnderstanding: "object", decisionSignificance: "array" },
|
||||
structuredResult: parsedResult,
|
||||
};
|
||||
|
||||
await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
|
||||
console.log("Live result written to: " + artifactPath);
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
// ─── CLI entry point ──────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
var args = process.argv.slice(2);
|
||||
|
||||
if (args.includes("--live")) {
|
||||
console.log("=== RTO.17A Three-Turn Separated Reasoning — live mode ===\n");
|
||||
var result = await executeLiveMode();
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
// Default: inspect-only, zero model calls
|
||||
var _a = await inspectApparatus();
|
||||
var inputCharacterCount = _a.inputCharacterCount;
|
||||
}
|
||||
|
||||
main().catch(function (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user