test(experiment): checkpoint borderline relationship control
This commit is contained in:
@@ -0,0 +1,629 @@
|
||||
/**
|
||||
* RTO.23A — Borderline Same-Topic Negative-Control Relationship-Discovery Apparatus
|
||||
*
|
||||
* Purpose: Test whether semantic relationship discovery distinguishes
|
||||
* topical similarity from material evidential bearing when both fragments
|
||||
* concern the same competitor, market, and industry context but the later
|
||||
* fragment does not materially advance the earlier uncertainty.
|
||||
*
|
||||
* Design boundary:
|
||||
* - Standalone experimental runner.
|
||||
* - Zero production code changes.
|
||||
* - Inspect-only by default (zero live model calls).
|
||||
* - --live flag enables exactly one live call producing a proposed relationship.
|
||||
*
|
||||
* Fixed earlier fragment:
|
||||
* RTO.18 Fragment 2 — competitor hiring + conference signals
|
||||
* (tests/experimental/results/rto-granular-fragment-turn2.json)
|
||||
*
|
||||
* Fixed borderline-control fragment:
|
||||
* EXPERIMENTAL_FIXTURE — same competitor, same market, same conference,
|
||||
* but only general commentary with no technical or product-development evidence.
|
||||
*
|
||||
* Output contract (same as RTO.21 / RTO.22):
|
||||
* { relationshipFound, relationship, remainingQualification }
|
||||
*/
|
||||
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");
|
||||
|
||||
// ─── Fragment paths ──────────────────────────────────────────────────────────
|
||||
|
||||
const FRAGMENT_2_PATH = path.resolve(
|
||||
__dirname,
|
||||
"../../tests/experimental/results/rto-granular-fragment-turn2.json"
|
||||
);
|
||||
|
||||
// ─── Borderline-control fragment (EXPERIMENTAL_FIXTURE) ──────────────────────
|
||||
// This fixture is deliberately topically close to Fragment 2 but lacks
|
||||
// any technical architecture, prototype, roadmap, launch timing, development
|
||||
// programme, or new hiring evidence. Those absences are the experimental
|
||||
// variable being tested.
|
||||
|
||||
const BORDERLINE_CONTROL_FRAGMENT = {
|
||||
targetNodeId: "borderline-control-competitor-market-presence",
|
||||
question: "What else has the competitor done publicly in this market?",
|
||||
observations: [
|
||||
"The competitor sponsored the same industry conference.",
|
||||
"Its chief executive gave opening remarks about growth in the market and the importance of solving the customer problem.",
|
||||
"The remarks contained no technical architecture, prototype, product roadmap, launch timing, development programme, or additional hiring information."
|
||||
],
|
||||
uncertainties: [],
|
||||
assumptions: [],
|
||||
relationships: [],
|
||||
possibleFollowUpQuestions: []
|
||||
};
|
||||
|
||||
// ─── Load fragment 2 ─────────────────────────────────────────────────────────
|
||||
|
||||
async function loadFragment2() {
|
||||
const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
|
||||
const frag2 = JSON.parse(frag2Raw);
|
||||
|
||||
if (!frag2.structuredResult) {
|
||||
throw new Error("Fragment 2 missing structuredResult");
|
||||
}
|
||||
|
||||
return {
|
||||
fragment: frag2,
|
||||
structuredResult: frag2.structuredResult
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Borderline-control integrity check ──────────────────────────────────────
|
||||
|
||||
function hasAffirmativeEvidence(text, terms) {
|
||||
// Check if any term appears as affirmative evidence (not in a negation).
|
||||
// A negation pattern is: "contained no X" / "no X present" / "without X".
|
||||
const lowerText = text.toLowerCase();
|
||||
|
||||
// Find all occurrences of each term and check whether they are inside
|
||||
// a negation scope. We look for the nearest preceding negation marker
|
||||
// that is closer than the next affirmative context breaker (a period,
|
||||
// semicolon, or newline).
|
||||
for (const term of terms) {
|
||||
let searchPos = 0;
|
||||
while (true) {
|
||||
const idx = lowerText.indexOf(term, searchPos);
|
||||
if (idx === -1) break;
|
||||
|
||||
// Find the nearest preceding negation marker
|
||||
let foundNegation = false;
|
||||
for (const marker of ["contained no", "not ", "no ", "without", "lacks"]) {
|
||||
const markerIdx = lowerText.lastIndexOf(marker, idx - 1);
|
||||
if (markerIdx !== -1) {
|
||||
// Verify no sentence-breaker between the marker and the term
|
||||
const between = lowerText.substring(markerIdx + marker.length, idx);
|
||||
if (!between.match(/[.;\n]/)) {
|
||||
foundNegation = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (foundNegation) {
|
||||
// Skip past this occurrence — it's negated
|
||||
searchPos = idx + term.length;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look forward to ensure the term is a complete word (not part of another word)
|
||||
const after = lowerText.substring(idx);
|
||||
if (after.match(/[^a-z]/) !== null) {
|
||||
return true; // Affirmative evidence found
|
||||
}
|
||||
searchPos = idx + 1;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function verifyBorderlineControlClean() {
|
||||
const text = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT);
|
||||
|
||||
// Same competitor / market / conference context should be present
|
||||
const lowerText = text.toLowerCase();
|
||||
const sameCompetitorContext = lowerText.includes("competitor");
|
||||
const sameMarketContext = lowerText.includes("market");
|
||||
const sameConferenceContext = lowerText.includes("conference");
|
||||
const customerProblemRef = lowerText.includes("customer problem");
|
||||
|
||||
// These must NOT have affirmative evidence (no material advancement of earlier uncertainty)
|
||||
const technicalArchitecture = hasAffirmativeEvidence(text, ["technical architecture"]);
|
||||
const prototypeEvidence = hasAffirmativeEvidence(text, ["prototype"]);
|
||||
const productRoadmap = hasAffirmativeEvidence(text, ["product roadmap"]);
|
||||
const launchTiming = hasAffirmativeEvidence(text, ["launch timing", "launch date"]);
|
||||
const developmentProgramme = hasAffirmativeEvidence(text, ["development programme"]);
|
||||
const newHiringEvidence = hasAffirmativeEvidence(text, ["additional hiring"]);
|
||||
|
||||
return {
|
||||
sameCompetitorContext,
|
||||
sameMarketContext,
|
||||
sameConferenceContext,
|
||||
customerProblemRef,
|
||||
technicalArchitecture,
|
||||
prototypeEvidence,
|
||||
productRoadmap,
|
||||
launchTiming,
|
||||
developmentProgramme,
|
||||
newHiringEvidence,
|
||||
clean: !technicalArchitecture && !prototypeEvidence && !productRoadmap && !launchTiming && !developmentProgramme && !newHiringEvidence
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Build the relationship-discovery prompt ──────────────────────────────────
|
||||
|
||||
function buildRelationshipPrompt(srs2, controlSrs) {
|
||||
const parts = [];
|
||||
|
||||
// Instructions — strict semantic discipline (same as RTO.21 / RTO.22)
|
||||
parts.push("You are examining two independent reasoning fragments produced during a");
|
||||
parts.push("competitive investigation. Your task is to determine whether the later");
|
||||
parts.push("fragment contains evidence that materially bears on an uncertainty,");
|
||||
parts.push("assumption or observation in the earlier fragment.");
|
||||
parts.push("");
|
||||
parts.push("CRITICAL RULES:");
|
||||
parts.push("- Only consider what is explicitly stated in these two fragments.");
|
||||
parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
|
||||
parts.push("- Do not infer facts not present in either fragment.");
|
||||
parts.push("- Do not rewrite or summarise either fragment.");
|
||||
parts.push("- Do not recommend an action or choose what to investigate next.");
|
||||
parts.push("");
|
||||
parts.push("SEMANTIC DISCIPLINE:");
|
||||
parts.push("- Identify only a relationship where the later fragment materially bears");
|
||||
parts.push(" on an uncertainty, assumption or observation in the earlier fragment.");
|
||||
parts.push("- Topical similarity alone is not sufficient.");
|
||||
parts.push("- Shared entities, market, industry or subject matter do not by themselves");
|
||||
parts.push(" constitute an evidential relationship.");
|
||||
parts.push("- Do not invent a connection merely because both fragments concern the");
|
||||
parts.push(" same wider situation.");
|
||||
parts.push("- If the later fragment does not materially change, qualify, support,");
|
||||
parts.push(" weaken or contradict an earlier item, return relationshipFound=false.");
|
||||
parts.push("");
|
||||
|
||||
// Fragment 2 (earlier)
|
||||
parts.push("=== Fragment 2 (earlier evidence) ===");
|
||||
parts.push(`Question: ${srs2.question}`);
|
||||
if (srs2.observations && srs2.observations.length) {
|
||||
parts.push("Observations:");
|
||||
for (const o of srs2.observations) {
|
||||
parts.push(` - ${o}`);
|
||||
}
|
||||
}
|
||||
if (srs2.uncertainties && srs2.uncertainties.length) {
|
||||
parts.push("Uncertainties:");
|
||||
for (const u of srs2.uncertainties) {
|
||||
parts.push(` - ${u}`);
|
||||
}
|
||||
}
|
||||
if (srs2.assumptions && srs2.assumptions.length) {
|
||||
parts.push("Assumptions:");
|
||||
for (const a of srs2.assumptions) {
|
||||
parts.push(` - ${a}`);
|
||||
}
|
||||
}
|
||||
if (srs2.relationships && srs2.relationships.length) {
|
||||
parts.push("Relationships (internal to fragment):");
|
||||
for (const r of srs2.relationships) {
|
||||
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
|
||||
}
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
// Borderline-control fragment (later)
|
||||
parts.push("=== Fragment (later evidence) ===");
|
||||
parts.push(`Question: ${controlSrs.question}`);
|
||||
if (controlSrs.observations && controlSrs.observations.length) {
|
||||
parts.push("Observations:");
|
||||
for (const o of controlSrs.observations) {
|
||||
parts.push(` - ${o}`);
|
||||
}
|
||||
}
|
||||
if (controlSrs.uncertainties && controlSrs.uncertainties.length) {
|
||||
parts.push("Uncertainties:");
|
||||
for (const u of controlSrs.uncertainties) {
|
||||
parts.push(` - ${u}`);
|
||||
}
|
||||
}
|
||||
if (controlSrs.assumptions && controlSrs.assumptions.length) {
|
||||
parts.push("Assumptions:");
|
||||
for (const a of controlSrs.assumptions) {
|
||||
parts.push(` - ${a}`);
|
||||
}
|
||||
}
|
||||
if (controlSrs.relationships && controlSrs.relationships.length) {
|
||||
parts.push("Relationships (internal to fragment):");
|
||||
for (const r of controlSrs.relationships) {
|
||||
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
|
||||
}
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
// Core question
|
||||
parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,");
|
||||
parts.push("assumption or observation in the earlier fragment?");
|
||||
parts.push("");
|
||||
|
||||
// Return format — same contract as RTO.21 / RTO.22
|
||||
parts.push('Return exactly one JSON object:');
|
||||
parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
|
||||
parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
// ─── Output contract validation (same as RTO.21 / RTO.22) ─────────────────────
|
||||
|
||||
function validateRelationshipOutput(result) {
|
||||
const errors = [];
|
||||
|
||||
if (typeof result.relationshipFound !== "boolean") {
|
||||
errors.push("relationshipFound must be a boolean");
|
||||
}
|
||||
|
||||
if (!result.relationshipFound) {
|
||||
if (result.relationship !== null && result.relationship !== undefined) {
|
||||
errors.push("When relationshipFound=false, relationship must be null or undefined");
|
||||
}
|
||||
if (!Array.isArray(result.remainingQualification)) {
|
||||
errors.push("remainingQualification must be an array when relationshipFound=false");
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
// When true, relationship object is required
|
||||
if (!result.relationship || typeof result.relationship !== "object") {
|
||||
errors.push("When relationshipFound=true, relationship must be an object");
|
||||
return errors;
|
||||
}
|
||||
|
||||
const rel = result.relationship;
|
||||
if (typeof rel.laterEvidence !== "string") {
|
||||
errors.push("relationship.laterEvidence must be a string");
|
||||
}
|
||||
if (typeof rel.earlierItem !== "string") {
|
||||
errors.push("relationship.earlierItem must be a string");
|
||||
}
|
||||
if (typeof rel.bearing !== "string") {
|
||||
errors.push("relationship.bearing must be a string");
|
||||
}
|
||||
|
||||
if (!Array.isArray(result.remainingQualification)) {
|
||||
errors.push("remainingQualification must be an array when relationshipFound=true");
|
||||
} else if (result.remainingQualification.some((s) => typeof s !== "string")) {
|
||||
errors.push("remainingQualification items must all be strings");
|
||||
}
|
||||
|
||||
// No forbidden fields (same as RTO.21 / RTO.22)
|
||||
const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
|
||||
"recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
|
||||
for (const field of forbidden) {
|
||||
if (field in result && typeof result[field] !== "undefined") {
|
||||
errors.push(`Forbidden field present: ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
// ─── Prompt-leak check ──────────────────────────────────────────────────────
|
||||
|
||||
function checkPromptLeak(prompt) {
|
||||
const lowerPrompt = prompt.toLowerCase();
|
||||
const leakTerms = [
|
||||
"negative control",
|
||||
"borderline control",
|
||||
"expected false",
|
||||
"no relationship expected",
|
||||
"rto.22",
|
||||
"manual answer"
|
||||
];
|
||||
|
||||
const found = [];
|
||||
for (const term of leakTerms) {
|
||||
if (lowerPrompt.includes(term.toLowerCase())) {
|
||||
found.push(term);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
leaked: found.length > 0,
|
||||
terms: found
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Inspect mode (default: zero model calls) ────────────────────────────────
|
||||
|
||||
async function inspectApparatus() {
|
||||
console.log("=== RTO.23A Borderline Same-Topic Relationship-Control Apparatus (inspect-only) ===\n");
|
||||
|
||||
// Load and verify fragment 2
|
||||
let structuredResult2;
|
||||
try {
|
||||
const frag2 = await loadFragment2();
|
||||
structuredResult2 = frag2.structuredResult;
|
||||
} catch (e) {
|
||||
console.log("ERROR: Cannot load Fragment 2:", e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Borderline-control integrity check
|
||||
const controlCheck = verifyBorderlineControlClean();
|
||||
console.log("--- Borderline-control fixture integrity ---");
|
||||
console.log(`Same competitor/market context present: ${controlCheck.sameCompetitorContext ? "YES" : "NO"}`);
|
||||
console.log(`Same conference context present: ${controlCheck.sameConferenceContext ? "YES" : "NO"}`);
|
||||
console.log(`General customer-problem reference present: ${controlCheck.customerProblemRef ? "YES" : "NO"}`);
|
||||
console.log(`Technical architecture present: ${controlCheck.technicalArchitecture ? "YES" : "NO"}`);
|
||||
console.log(`Prototype evidence present: ${controlCheck.prototypeEvidence ? "YES" : "NO"}`);
|
||||
console.log(`Product roadmap present: ${controlCheck.productRoadmap ? "YES" : "NO"}`);
|
||||
console.log(`Launch timing evidence present: ${controlCheck.launchTiming ? "YES" : "NO"}`);
|
||||
console.log(`Development programme present: ${controlCheck.developmentProgramme ? "YES" : "NO"}`);
|
||||
console.log(`New hiring evidence present: ${controlCheck.newHiringEvidence ? "YES" : "NO"}`);
|
||||
|
||||
if (!controlCheck.clean) {
|
||||
console.log("\nBORDERLINE_CONTROL_CLEAN:\nNO");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("\nBORDERLINE_CONTROL_CLEAN:\nYES");
|
||||
|
||||
// Fragment verification
|
||||
const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
|
||||
console.log("\n--- Fragment verification ---");
|
||||
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||
console.log(`Borderline-control fragment supplied: YES`);
|
||||
|
||||
// Build prompt for inspection
|
||||
const prompt = buildRelationshipPrompt(structuredResult2, BORDERLINE_CONTROL_FRAGMENT);
|
||||
|
||||
// Measure character sizes
|
||||
const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
|
||||
const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
|
||||
const controlStructuredText = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2);
|
||||
|
||||
console.log("\n--- Context-size measurement ---");
|
||||
console.log(`instructionCharacterCount: ${instructionPart.length}`);
|
||||
console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
|
||||
console.log(`borderlineControlCharacterCount: ${controlStructuredText.length}`);
|
||||
console.log(`inputCharacterCount: ${prompt.length}`);
|
||||
|
||||
// ── Boundary checks ───────────────────────────────────────────
|
||||
|
||||
console.log("\n--- Input boundary verification ---");
|
||||
|
||||
const fragment3Supplied = prompt.includes("Fragment 3");
|
||||
console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);
|
||||
|
||||
const rto21PositiveResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
|
||||
prompt.includes("active competing-product development more plausible");
|
||||
console.log(`RTO.21 positive result supplied: ${rto21PositiveResultSupplied ? "YES" : "NO"}`);
|
||||
|
||||
const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
|
||||
prompt.includes("whole-case");
|
||||
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
const centralCaseInPrompt = prompt.includes("central case") ||
|
||||
prompt.includes("centralStatement");
|
||||
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
const currentViewInPrompt = prompt.includes("derived view") ||
|
||||
prompt.includes("focused view") || prompt.includes("current focused");
|
||||
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
|
||||
prompt.includes("other fragments") || prompt.includes("additional fragment");
|
||||
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
const turnHistoryInPrompt = prompt.includes("turn history") ||
|
||||
prompt.includes("prior turns") || prompt.includes("conversation history");
|
||||
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
// ── Prompt-leak check (mandatory) ─────────────────────────────
|
||||
|
||||
const leakCheck = checkPromptLeak(prompt);
|
||||
console.log("\n--- Prompt-leak check ---");
|
||||
console.log(`EXPECTED_OUTCOME_LEAKED_TO_MODEL:\n${leakCheck.leaked ? "YES" : "NO"}`);
|
||||
if (leakCheck.leaked) {
|
||||
console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`);
|
||||
}
|
||||
|
||||
// ── Output contract validation capability ─────────────────────
|
||||
|
||||
const mockProposal = {
|
||||
relationshipFound: true,
|
||||
relationship: {
|
||||
laterEvidence: "test",
|
||||
earlierItem: "test",
|
||||
bearing: "test",
|
||||
},
|
||||
remainingQualification: ["test"],
|
||||
};
|
||||
const mockNoRel = {
|
||||
relationshipFound: false,
|
||||
relationship: null,
|
||||
remainingQualification: [],
|
||||
};
|
||||
const proposalErrors = validateRelationshipOutput(mockProposal);
|
||||
const noRelErrors = validateRelationshipOutput(mockNoRel);
|
||||
console.log("\n--- Output contract (same as RTO.21) ---");
|
||||
console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
|
||||
console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
|
||||
console.log(`relationshipFound=false allowed: YES`);
|
||||
console.log(`Topical similarity explicitly insufficient: YES`);
|
||||
|
||||
// ── Live route ────────────────────────────────────────────────
|
||||
console.log("\n--- Live route ---");
|
||||
console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live");
|
||||
|
||||
// ── Environment check ─────────────────────────────────────────
|
||||
const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
|
||||
const modelConfigured = !!process.env.OLLAMA_MODEL;
|
||||
console.log("\n--- Environment ---");
|
||||
console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`);
|
||||
console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);
|
||||
|
||||
// ── Inspect summary ───────────────────────────────────────────
|
||||
const allFragmentsOK = f2Loaded;
|
||||
const noBoundaryLeaks = !fragment3Supplied && !rto21PositiveResultSupplied &&
|
||||
!wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt &&
|
||||
!otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean && !leakCheck.leaked;
|
||||
|
||||
console.log("\n=== SUMMARY ===");
|
||||
console.log(`Inspect live calls: 0`);
|
||||
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||
console.log(`Borderline-control fragment supplied: YES`);
|
||||
console.log(`Fragment 3 supplied: NO`);
|
||||
console.log(`RTO.21 positive result supplied: NO`);
|
||||
console.log(`RTO.22 negative result supplied: NO`);
|
||||
console.log(`Known relationship supplied: NO`);
|
||||
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`BORDERLINE_CONTROL_CLEAN:\nYES`);
|
||||
console.log(`--live route exposed: YES`);
|
||||
console.log(`maximum live calls per --live: 1`);
|
||||
console.log(`relationshipFound=false allowed: YES`);
|
||||
|
||||
const passed = allFragmentsOK && noBoundaryLeaks;
|
||||
console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
|
||||
|
||||
if (!passed) {
|
||||
console.log("\nApparatus cannot satisfy boundary without production changes.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return { prompt };
|
||||
}
|
||||
|
||||
// ─── Live execution (exactly one model call) ──────────────────────────────────
|
||||
|
||||
async function executeLive() {
|
||||
const baseUrl = process.env.OLLAMA_BASE_URL;
|
||||
if (!baseUrl) {
|
||||
console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode.");
|
||||
process.exit(1);
|
||||
}
|
||||
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||
console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const modelName = process.env.OLLAMA_MODEL;
|
||||
if (!modelName) {
|
||||
console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Load fragment 2 and build prompt
|
||||
const { structuredResult: srs2 } = await loadFragment2();
|
||||
const prompt = buildRelationshipPrompt(srs2, BORDERLINE_CONTROL_FRAGMENT);
|
||||
|
||||
// Prompt-leak check before sending
|
||||
const leakCheck = checkPromptLeak(prompt);
|
||||
if (leakCheck.leaked) {
|
||||
console.error(`PROMPT_LEAK: Expected outcome leaked to model input via: ${leakCheck.terms.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Prepare artifact path for live result
|
||||
await fs.mkdir(resultsDir, { recursive: true });
|
||||
|
||||
const startedAt = Date.now();
|
||||
console.log("Live mode: sending to model...");
|
||||
|
||||
const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
|
||||
const raw = await provider.generateReconstruction(prompt, modelName);
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
|
||||
console.log(`Model response received in ${elapsedMs}ms`);
|
||||
|
||||
// Provider returns a parsed JavaScript object via recoverJson().
|
||||
const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => {
|
||||
console.error("Provider returned unexpected type (expected parsed object):", typeof raw);
|
||||
process.exit(1);
|
||||
})();
|
||||
|
||||
// Validate output contract
|
||||
const errors = validateRelationshipOutput(parsedResult);
|
||||
if (errors.length > 0) {
|
||||
console.error("\nLive output contract violations:");
|
||||
for (const err of errors) {
|
||||
console.error(` - ${err}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Write artifact
|
||||
const artifactPath = path.resolve(
|
||||
resultsDir,
|
||||
"rto-fragment-relationship-borderline-control-live.json"
|
||||
);
|
||||
|
||||
const controlCheck = verifyBorderlineControlClean();
|
||||
const payload = {
|
||||
apparatus: "rto-fragment-relationship-borderline-control.mjs",
|
||||
experiment: "RTO.23A",
|
||||
artifactType: "LIVE RESULT — Borderline same-topic relationship discovery",
|
||||
modelName: modelName,
|
||||
elapsedMs: elapsedMs,
|
||||
inputCharacterCount: prompt.length,
|
||||
instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
|
||||
fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
|
||||
borderlineControlCharacterCount: JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2).length,
|
||||
fragment2Path: FRAGMENT_2_PATH,
|
||||
controlFixtureType: "EXPERIMENTAL_FIXTURE",
|
||||
sameCompetitorContext: controlCheck.sameCompetitorContext,
|
||||
sameConferenceContext: controlCheck.sameConferenceContext,
|
||||
borderlineControlClean: controlCheck.clean,
|
||||
knownRelationshipSuppliedToModel: false,
|
||||
wholeSituationGraphSupplied: false,
|
||||
centralCaseStatementSupplied: false,
|
||||
turnHistorySupplied: false,
|
||||
proposedRelationship: parsedResult,
|
||||
};
|
||||
|
||||
await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
|
||||
console.log(`\nLive result written to: ${artifactPath}`);
|
||||
console.log("\n=== Proposed Relationship ===");
|
||||
console.log(JSON.stringify(parsedResult, null, 2));
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
// ─── CLI entry point ──────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
// --live flag for exactly one live model call
|
||||
if (args.includes("--live")) {
|
||||
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
|
||||
console.log("WARNING: This will make exactly ONE live model call.\n");
|
||||
const result = await executeLive();
|
||||
return result;
|
||||
}
|
||||
|
||||
// Default: inspect-only, zero model calls
|
||||
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
|
||||
console.log("Run with --live for one live model call.\n");
|
||||
|
||||
const result = await inspectApparatus();
|
||||
if (result === null) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user