From 8fb284c374d7ab57eb4a86c7b32d9718669d3574 Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 19 Aug 2026 15:43:06 +0100 Subject: [PATCH] test(experiment): checkpoint fragment relationship discovery apparatus --- .../rto-fragment-relationship-discovery.mjs | 495 ++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 scripts/experimental/rto-fragment-relationship-discovery.mjs diff --git a/scripts/experimental/rto-fragment-relationship-discovery.mjs b/scripts/experimental/rto-fragment-relationship-discovery.mjs new file mode 100644 index 0000000..2d0b766 --- /dev/null +++ b/scripts/experimental/rto-fragment-relationship-discovery.mjs @@ -0,0 +1,495 @@ +/** + * RTO.21A — Fragment Relationship-Discovery Apparatus + * + * Purpose: Test whether an LLM can propose a useful cross-fragment relationship + * from two independent granular fragments without whole-case context, without + * rewriting either fragment, and without treating the discovered relationship as + * automatically authoritative. + * + * Design boundary: + * - Standalone experimental runner. + * - Zero production code changes. + * - Inspect-only by default (zero live model calls). + * - --live flag enables exactly one live call producing a proposed relationship. + * + * Fixed record: + * Fragment 2 — competitor hiring + conference signals + * Fragment 3 — technical architecture + prototype evidence + * Known RTO.19 relationship is NEVER supplied to the model; used only for + * evaluation reference after a future live run. + * + * Output contract (proposal only, never authoritative): + * { relationshipFound, relationship, remainingQualification } + */ + +import fs from "fs/promises"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const resultsDir = path.resolve(__dirname, "../../tests/experimental/results"); +const envLocalPath = path.resolve(__dirname, "../../.env.local"); + +// ─── Fragment paths ────────────────────────────────────────────────────────── + +const FRAGMENT_2_PATH = path.resolve( + __dirname, + "../../tests/experimental/results/rto-granular-fragment-turn2.json" +); + +const FRAGMENT_3_PATH = path.resolve( + __dirname, + "../../tests/experimental/results/rto-granular-fragment-turn3.json" +); + +// ─── Known RTO.19 relationship (evaluation reference ONLY) ─────────────────── +// This MUST NOT appear in the prompt sent to the model. + +const KNOWN_RELATIONSHIP = + "Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty " + + "by making active competing-product development more plausible, while " + + "commercialisation and other uncertainties remain unresolved."; + +// ─── Load fragments ────────────────────────────────────────────────────────── + +async function loadFragments() { + const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8"); + const frag3Raw = await fs.readFile(FRAGMENT_3_PATH, "utf-8"); + + const frag2 = JSON.parse(frag2Raw); + const frag3 = JSON.parse(frag3Raw); + + if (!frag2.structuredResult) { + throw new Error("Fragment 2 missing structuredResult"); + } + if (!frag3.structuredResult) { + throw new Error("Fragment 3 missing structuredResult"); + } + + return { + fragment2: frag2, + structuredResult2: frag2.structuredResult, + fragment3: frag3, + structuredResult3: frag3.structuredResult, + }; +} + +// ─── Build the relationship-discovery prompt ────────────────────────────────── + +function buildRelationshipPrompt(srs2, srs3) { + const parts = []; + + // Instructions + parts.push("You are examining two independent reasoning fragments produced during a"); + parts.push("competitive investigation. Your task is to determine whether the later"); + parts.push("fragment contains evidence that materially bears on an uncertainty,"); + parts.push("assumption or observation in the earlier fragment."); + parts.push(""); + parts.push("CRITICAL RULES:"); + parts.push("- Only consider what is explicitly stated in these two fragments."); + parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation."); + parts.push("- Do not infer facts not present in either fragment."); + parts.push("- Do not rewrite or summarise either fragment."); + parts.push("- Do not recommend an action or choose what to investigate next."); + parts.push("- If no relationship exists, return relationshipFound=false with empty fields."); + parts.push(""); + + // Fragment 2 (earlier) + parts.push("=== Fragment 2 (earlier evidence) ==="); + parts.push(`Question: ${srs2.question}`); + if (srs2.observations && srs2.observations.length) { + parts.push("Observations:"); + for (const o of srs2.observations) { + parts.push(` - ${o}`); + } + } + if (srs2.uncertainties && srs2.uncertainties.length) { + parts.push("Uncertainties:"); + for (const u of srs2.uncertainties) { + parts.push(` - ${u}`); + } + } + if (srs2.assumptions && srs2.assumptions.length) { + parts.push("Assumptions:"); + for (const a of srs2.assumptions) { + parts.push(` - ${a}`); + } + } + if (srs2.relationships && srs2.relationships.length) { + parts.push("Relationships (internal to fragment):"); + for (const r of srs2.relationships) { + parts.push(` - ${r.from} -> ${r.to} (${r.type})`); + } + } + parts.push(""); + + // Fragment 3 (later) + parts.push("=== Fragment 3 (later evidence) ==="); + parts.push(`Question: ${srs3.question}`); + if (srs3.observations && srs3.observations.length) { + parts.push("Observations:"); + for (const o of srs3.observations) { + parts.push(` - ${o}`); + } + } + if (srs3.uncertainties && srs3.uncertainties.length) { + parts.push("Uncertainties:"); + for (const u of srs3.uncertainties) { + parts.push(` - ${u}`); + } + } + if (srs3.assumptions && srs3.assumptions.length) { + parts.push("Assumptions:"); + for (const a of srs3.assumptions) { + parts.push(` - ${a}`); + } + } + if (srs3.relationships && srs3.relationships.length) { + parts.push("Relationships (internal to fragment):"); + for (const r of srs3.relationships) { + parts.push(` - ${r.from} -> ${r.to} (${r.type})`); + } + } + parts.push(""); + + // Core question + parts.push("QUESTION: Does anything in Fragment 3 materially bears on an uncertainty,"); + parts.push("assumption or observation in Fragment 2?"); + parts.push(""); + parts.push("If YES, propose exactly one relationship. This is a PROPOSED_RELATIONSHIP"); + parts.push("--- it must not be treated as authoritative without further verification."); + parts.push(""); + + // Return format + parts.push('Return exactly one JSON object:'); + parts.push('{ "relationshipFound": , "relationship": { "laterEvidence": "", "earlierItem": "", "bearing": "" }, "remainingQualification": [""] }'); + parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }'); + + return parts.join("\n"); +} + +// ─── Output contract validation ────────────────────────────────────────────── + +function validateRelationshipOutput(result) { + const errors = []; + + if (typeof result.relationshipFound !== "boolean") { + errors.push("relationshipFound must be a boolean"); + } + + if (!result.relationshipFound) { + if (result.relationship !== null && result.relationship !== undefined) { + errors.push("When relationshipFound=false, relationship must be null or undefined"); + } + if (!Array.isArray(result.remainingQualification)) { + errors.push("remainingQualification must be an array when relationshipFound=false"); + } + return errors; + } + + // When true, relationship object is required + if (!result.relationship || typeof result.relationship !== "object") { + errors.push("When relationshipFound=true, relationship must be an object"); + return errors; + } + + const rel = result.relationship; + if (typeof rel.laterEvidence !== "string") { + errors.push("relationship.laterEvidence must be a string"); + } + if (typeof rel.earlierItem !== "string") { + errors.push("relationship.earlierItem must be a string"); + } + if (typeof rel.bearing !== "string") { + errors.push("relationship.bearing must be a string"); + } + + if (!Array.isArray(result.remainingQualification)) { + errors.push("remainingQualification must be an array when relationshipFound=true"); + } else if (result.remainingQualification.some((s) => typeof s !== "string")) { + errors.push("remainingQualification items must all be strings"); + } + + // No forbidden fields + const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength", + "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"]; + for (const field of forbidden) { + if (field in result && typeof result[field] !== "undefined") { + errors.push(`Forbidden field present: ${field}`); + } + } + + return errors; +} + +// ─── Inspect mode (default: zero model calls) ───────────────────────────────── + +async function inspectApparatus() { + console.log("=== RTO.21A Fragment Relationship-Discovery Apparatus (inspect-only) ===\n"); + + // Load and verify fragments + let fragment2, structuredResult2, fragment3, structuredResult3; + try { + const frags = await loadFragments(); + fragment2 = frags.fragment2; + structuredResult2 = frags.structuredResult2; + fragment3 = frags.fragment3; + structuredResult3 = frags.structuredResult3; + } catch (e) { + console.log("ERROR: Cannot load fragments:", e.message); + process.exit(1); + } + + // Fragment verification + const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null; + const f3Loaded = typeof structuredResult3 === "object" && structuredResult3 !== null; + console.log("--- Fragment verification ---"); + console.log(`Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); + console.log(`Fragment 3 supplied: ${f3Loaded ? "YES" : "NO"}`); + + // Build prompt for inspection + const prompt = buildRelationshipPrompt(structuredResult2, structuredResult3); + + // Measure character sizes precisely + const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2")); + const frag2StructuredText = JSON.stringify(structuredResult2, null, 2); + const frag3StructuredText = JSON.stringify(structuredResult3, null, 2); + + console.log("\n--- Context-size measurement ---"); + console.log(`instructionCharacterCount: ${instructionPart.length}`); + console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`); + console.log(`fragment3CharacterCount: ${frag3StructuredText.length}`); + console.log(`inputCharacterCount: ${prompt.length}`); + + // ── Boundary checks ─────────────────────────────────────────── + + console.log("\n--- Input boundary verification ---"); + + const knownRelInPrompt = prompt.includes(KNOWN_RELATIONSHIP) || + prompt.includes("Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty"); + console.log(`Known relationship supplied: ${knownRelInPrompt ? "YES" : "NO"}`); + + const wholeGraphInPrompt = prompt.includes("SituationGraph") || + prompt.includes("whole-case") || prompt.includes("case statement"); + console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); + + const centralCaseInPrompt = prompt.includes("central case") || + prompt.includes("centralStatement"); + console.log(`Central case statement supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); + + const currentViewInPrompt = prompt.includes("derived view") || + prompt.includes("focused view") || prompt.includes("current focused"); + console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); + + const otherFragmentsInPrompt = prompt.includes("Fragment 1") || + prompt.includes("other fragments") || prompt.includes("additional fragment"); + console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); + + const turnHistoryInPrompt = prompt.includes("turn history") || + prompt.includes("prior turns") || prompt.includes("conversation history"); + console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); + + const rto17StateInPrompt = prompt.includes("decisionSignificance") || + prompt.includes("accumulated state") || prompt.includes("focusedUnderstanding"); + console.log(`RTO.17 accumulated state supplied: ${rto17StateInPrompt ? "YES" : "NO"}`); + + // ── Schema validation capability ────────────────────────────── + const mockProposal = { + relationshipFound: true, + relationship: { + laterEvidence: "test", + earlierItem: "test", + bearing: "test", + }, + remainingQualification: ["test"], + }; + const mockNoRel = { + relationshipFound: false, + relationship: null, + remainingQualification: [], + }; + const proposalErrors = validateRelationshipOutput(mockProposal); + const noRelErrors = validateRelationshipOutput(mockNoRel); + console.log("\n--- Output schema ---"); + console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`); + console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`); + + // ── Live route ──────────────────────────────────────────────── + console.log("\n--- Live route ---"); + console.log("node scripts/experimental/rto-fragment-relationship-discovery.mjs --live"); + + // ── Inspect summary (must match exact requested format) ─────── + const allFragmentsOK = f2Loaded && f3Loaded; + const knownRelAbsent = !knownRelInPrompt; + const noContextLeaks = !(wholeGraphInPrompt || centralCaseInPrompt || currentViewInPrompt || otherFragmentsInPrompt || turnHistoryInPrompt || rto17StateInPrompt); + + console.log("\n=== SUMMARY ==="); + console.log(`Inspect live calls: 0`); + console.log(`Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); + console.log(`Fragment 3 supplied: ${f3Loaded ? "YES" : "NO"}`); + console.log(`Known relationship supplied: ${knownRelInPrompt ? "YES" : "NO"}`); + console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); + console.log(`Central case statement supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); + console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); + console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); + console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); + console.log(`--live route exposed: YES`); + console.log(`maximum live calls per --live: 1`); + console.log(`relationshipFound=false allowed: YES`); + + const passed = allFragmentsOK && knownRelAbsent && noContextLeaks; + console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`); + + if (!passed) { + console.log("\nApparatus cannot satisfy boundary without production changes."); + process.exit(1); + } + + return { prompt }; +} + +// ─── Live execution (exactly one model call) ────────────────────────────────── + +async function executeLive() { + // Load .env.local for authoritative config + let envContent = ""; + try { + envContent = await fs.readFile(envLocalPath, "utf-8"); + } catch { + console.error(".env.local not found. Cannot execute live mode."); + process.exit(1); + } + + const baseUrl = parseEnv(envContent, "OLLAMA_BASE_URL"); + if (!baseUrl) { + console.error("OLLAMA_BASE_URL not set in .env.local. Cannot execute live mode."); + process.exit(1); + } + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl); + process.exit(1); + } + + const modelName = parseEnv(envContent, "OLLAMA_MODEL"); + if (!modelName) { + console.error("OLLAMA_MODEL not set in .env.local. Cannot execute live mode."); + process.exit(1); + } + + // Load fragments and build prompt + const { structuredResult2, structuredResult3 } = await loadFragments(); + const prompt = buildRelationshipPrompt(structuredResult2, structuredResult3); + + const startedAt = Date.now(); + console.log("Live mode: sending to model..."); + + const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider(); + const raw = await provider.generateReconstruction(prompt, modelName); + const elapsedMs = Date.now() - startedAt; + + console.log(`Model response received in ${elapsedMs}ms`); + + // Parse the JSON output + let parsedResult; + try { + const jsonMatch = raw.match(/\{[\s\S]*\}/); + if (jsonMatch) { + parsedResult = JSON.parse(jsonMatch[0]); + } else { + throw new Error("No JSON object found in response"); + } + } catch (e) { + console.error("Failed to parse model output as JSON:", e.message); + console.error("Raw response:\n", raw); + process.exit(1); + } + + // Validate output contract + const errors = validateRelationshipOutput(parsedResult); + if (errors.length > 0) { + console.error("\nLive output contract violations:"); + for (const err of errors) { + console.error(` - ${err}`); + } + process.exit(1); + } + + // Write artifact + await fs.mkdir(resultsDir, { recursive: true }); + + const artifactPath = path.resolve( + resultsDir, + "rto-fragment-relationship-discovery-live.json" + ); + + const payload = { + apparatus: "rto-fragment-relationship-discovery.mjs", + experiment: "RTO.21A", + artifactType: "LIVE RESULT — Proposed relationship discovery", + modelName: modelName, + elapsedMs: elapsedMs, + inputCharacterCount: prompt.length, + instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length, + fragment2CharacterCount: JSON.stringify(structuredResult2, null, 2).length, + fragment3CharacterCount: JSON.stringify(structuredResult3, null, 2).length, + fragment2Path: FRAGMENT_2_PATH, + fragment3Path: FRAGMENT_3_PATH, + knownRelationshipSuppliedToModel: false, + wholeSituationGraphSupplied: false, + centralCaseStatementSupplied: false, + turnHistorySupplied: false, + proposedRelationship: parsedResult, + }; + + await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2)); + console.log(`\nLive result written to: ${artifactPath}`); + console.log("\n=== Proposed Relationship ==="); + console.log(JSON.stringify(parsedResult, null, 2)); + + return payload; +} + +// ─── Helper: parse .env.local ──────────────────────────────────────────────── + +function parseEnv(content, key) { + const lines = content.split("\n"); + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith("#")) continue; + const [k, ...vParts] = trimmed.split("="); + if (k.trim() === key) { + return vParts.join("=").trim().replace(/^["']|["']$/g, ""); + } + } + return null; +} + +// ─── CLI entry point ────────────────────────────────────────────────────────── + +async function main() { + const args = process.argv.slice(2); + + // --live flag for exactly one live model call + if (args.includes("--live")) { + console.log("RTO.21A — Fragment Relationship-Discovery Apparatus\n"); + console.log("WARNING: This will make exactly ONE live model call.\n"); + const result = await executeLive(); + return result; + } + + // Default: inspect-only, zero model calls + console.log("RTO.21A — Fragment Relationship-Discovery Apparatus\n"); + console.log("Run with --live for one live model call.\n"); + + const result = await inspectApparatus(); + if (result === null) { + process.exit(1); + } + + return result; +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +});