test(experiment): checkpoint borderline relationship control
This commit is contained in:
@@ -0,0 +1,629 @@
|
|||||||
|
/**
|
||||||
|
* RTO.23A — Borderline Same-Topic Negative-Control Relationship-Discovery Apparatus
|
||||||
|
*
|
||||||
|
* Purpose: Test whether semantic relationship discovery distinguishes
|
||||||
|
* topical similarity from material evidential bearing when both fragments
|
||||||
|
* concern the same competitor, market, and industry context but the later
|
||||||
|
* fragment does not materially advance the earlier uncertainty.
|
||||||
|
*
|
||||||
|
* Design boundary:
|
||||||
|
* - Standalone experimental runner.
|
||||||
|
* - Zero production code changes.
|
||||||
|
* - Inspect-only by default (zero live model calls).
|
||||||
|
* - --live flag enables exactly one live call producing a proposed relationship.
|
||||||
|
*
|
||||||
|
* Fixed earlier fragment:
|
||||||
|
* RTO.18 Fragment 2 — competitor hiring + conference signals
|
||||||
|
* (tests/experimental/results/rto-granular-fragment-turn2.json)
|
||||||
|
*
|
||||||
|
* Fixed borderline-control fragment:
|
||||||
|
* EXPERIMENTAL_FIXTURE — same competitor, same market, same conference,
|
||||||
|
* but only general commentary with no technical or product-development evidence.
|
||||||
|
*
|
||||||
|
* Output contract (same as RTO.21 / RTO.22):
|
||||||
|
* { relationshipFound, relationship, remainingQualification }
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from "fs/promises";
|
||||||
|
import path from "path";
|
||||||
|
import { fileURLToPath } from "url";
|
||||||
|
import dotenv from "dotenv";
|
||||||
|
|
||||||
|
dotenv.config({ path: ".env.local" });
|
||||||
|
|
||||||
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");
|
||||||
|
|
||||||
|
// ─── Fragment paths ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const FRAGMENT_2_PATH = path.resolve(
|
||||||
|
__dirname,
|
||||||
|
"../../tests/experimental/results/rto-granular-fragment-turn2.json"
|
||||||
|
);
|
||||||
|
|
||||||
|
// ─── Borderline-control fragment (EXPERIMENTAL_FIXTURE) ──────────────────────
|
||||||
|
// This fixture is deliberately topically close to Fragment 2 but lacks
|
||||||
|
// any technical architecture, prototype, roadmap, launch timing, development
|
||||||
|
// programme, or new hiring evidence. Those absences are the experimental
|
||||||
|
// variable being tested.
|
||||||
|
|
||||||
|
const BORDERLINE_CONTROL_FRAGMENT = {
|
||||||
|
targetNodeId: "borderline-control-competitor-market-presence",
|
||||||
|
question: "What else has the competitor done publicly in this market?",
|
||||||
|
observations: [
|
||||||
|
"The competitor sponsored the same industry conference.",
|
||||||
|
"Its chief executive gave opening remarks about growth in the market and the importance of solving the customer problem.",
|
||||||
|
"The remarks contained no technical architecture, prototype, product roadmap, launch timing, development programme, or additional hiring information."
|
||||||
|
],
|
||||||
|
uncertainties: [],
|
||||||
|
assumptions: [],
|
||||||
|
relationships: [],
|
||||||
|
possibleFollowUpQuestions: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// ─── Load fragment 2 ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function loadFragment2() {
|
||||||
|
const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
|
||||||
|
const frag2 = JSON.parse(frag2Raw);
|
||||||
|
|
||||||
|
if (!frag2.structuredResult) {
|
||||||
|
throw new Error("Fragment 2 missing structuredResult");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
fragment: frag2,
|
||||||
|
structuredResult: frag2.structuredResult
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Borderline-control integrity check ──────────────────────────────────────
|
||||||
|
|
||||||
|
function hasAffirmativeEvidence(text, terms) {
|
||||||
|
// Check if any term appears as affirmative evidence (not in a negation).
|
||||||
|
// A negation pattern is: "contained no X" / "no X present" / "without X".
|
||||||
|
const lowerText = text.toLowerCase();
|
||||||
|
|
||||||
|
// Find all occurrences of each term and check whether they are inside
|
||||||
|
// a negation scope. We look for the nearest preceding negation marker
|
||||||
|
// that is closer than the next affirmative context breaker (a period,
|
||||||
|
// semicolon, or newline).
|
||||||
|
for (const term of terms) {
|
||||||
|
let searchPos = 0;
|
||||||
|
while (true) {
|
||||||
|
const idx = lowerText.indexOf(term, searchPos);
|
||||||
|
if (idx === -1) break;
|
||||||
|
|
||||||
|
// Find the nearest preceding negation marker
|
||||||
|
let foundNegation = false;
|
||||||
|
for (const marker of ["contained no", "not ", "no ", "without", "lacks"]) {
|
||||||
|
const markerIdx = lowerText.lastIndexOf(marker, idx - 1);
|
||||||
|
if (markerIdx !== -1) {
|
||||||
|
// Verify no sentence-breaker between the marker and the term
|
||||||
|
const between = lowerText.substring(markerIdx + marker.length, idx);
|
||||||
|
if (!between.match(/[.;\n]/)) {
|
||||||
|
foundNegation = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (foundNegation) {
|
||||||
|
// Skip past this occurrence — it's negated
|
||||||
|
searchPos = idx + term.length;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look forward to ensure the term is a complete word (not part of another word)
|
||||||
|
const after = lowerText.substring(idx);
|
||||||
|
if (after.match(/[^a-z]/) !== null) {
|
||||||
|
return true; // Affirmative evidence found
|
||||||
|
}
|
||||||
|
searchPos = idx + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyBorderlineControlClean() {
|
||||||
|
const text = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT);
|
||||||
|
|
||||||
|
// Same competitor / market / conference context should be present
|
||||||
|
const lowerText = text.toLowerCase();
|
||||||
|
const sameCompetitorContext = lowerText.includes("competitor");
|
||||||
|
const sameMarketContext = lowerText.includes("market");
|
||||||
|
const sameConferenceContext = lowerText.includes("conference");
|
||||||
|
const customerProblemRef = lowerText.includes("customer problem");
|
||||||
|
|
||||||
|
// These must NOT have affirmative evidence (no material advancement of earlier uncertainty)
|
||||||
|
const technicalArchitecture = hasAffirmativeEvidence(text, ["technical architecture"]);
|
||||||
|
const prototypeEvidence = hasAffirmativeEvidence(text, ["prototype"]);
|
||||||
|
const productRoadmap = hasAffirmativeEvidence(text, ["product roadmap"]);
|
||||||
|
const launchTiming = hasAffirmativeEvidence(text, ["launch timing", "launch date"]);
|
||||||
|
const developmentProgramme = hasAffirmativeEvidence(text, ["development programme"]);
|
||||||
|
const newHiringEvidence = hasAffirmativeEvidence(text, ["additional hiring"]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
sameCompetitorContext,
|
||||||
|
sameMarketContext,
|
||||||
|
sameConferenceContext,
|
||||||
|
customerProblemRef,
|
||||||
|
technicalArchitecture,
|
||||||
|
prototypeEvidence,
|
||||||
|
productRoadmap,
|
||||||
|
launchTiming,
|
||||||
|
developmentProgramme,
|
||||||
|
newHiringEvidence,
|
||||||
|
clean: !technicalArchitecture && !prototypeEvidence && !productRoadmap && !launchTiming && !developmentProgramme && !newHiringEvidence
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Build the relationship-discovery prompt ──────────────────────────────────
|
||||||
|
|
||||||
|
function buildRelationshipPrompt(srs2, controlSrs) {
|
||||||
|
const parts = [];
|
||||||
|
|
||||||
|
// Instructions — strict semantic discipline (same as RTO.21 / RTO.22)
|
||||||
|
parts.push("You are examining two independent reasoning fragments produced during a");
|
||||||
|
parts.push("competitive investigation. Your task is to determine whether the later");
|
||||||
|
parts.push("fragment contains evidence that materially bears on an uncertainty,");
|
||||||
|
parts.push("assumption or observation in the earlier fragment.");
|
||||||
|
parts.push("");
|
||||||
|
parts.push("CRITICAL RULES:");
|
||||||
|
parts.push("- Only consider what is explicitly stated in these two fragments.");
|
||||||
|
parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
|
||||||
|
parts.push("- Do not infer facts not present in either fragment.");
|
||||||
|
parts.push("- Do not rewrite or summarise either fragment.");
|
||||||
|
parts.push("- Do not recommend an action or choose what to investigate next.");
|
||||||
|
parts.push("");
|
||||||
|
parts.push("SEMANTIC DISCIPLINE:");
|
||||||
|
parts.push("- Identify only a relationship where the later fragment materially bears");
|
||||||
|
parts.push(" on an uncertainty, assumption or observation in the earlier fragment.");
|
||||||
|
parts.push("- Topical similarity alone is not sufficient.");
|
||||||
|
parts.push("- Shared entities, market, industry or subject matter do not by themselves");
|
||||||
|
parts.push(" constitute an evidential relationship.");
|
||||||
|
parts.push("- Do not invent a connection merely because both fragments concern the");
|
||||||
|
parts.push(" same wider situation.");
|
||||||
|
parts.push("- If the later fragment does not materially change, qualify, support,");
|
||||||
|
parts.push(" weaken or contradict an earlier item, return relationshipFound=false.");
|
||||||
|
parts.push("");
|
||||||
|
|
||||||
|
// Fragment 2 (earlier)
|
||||||
|
parts.push("=== Fragment 2 (earlier evidence) ===");
|
||||||
|
parts.push(`Question: ${srs2.question}`);
|
||||||
|
if (srs2.observations && srs2.observations.length) {
|
||||||
|
parts.push("Observations:");
|
||||||
|
for (const o of srs2.observations) {
|
||||||
|
parts.push(` - ${o}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (srs2.uncertainties && srs2.uncertainties.length) {
|
||||||
|
parts.push("Uncertainties:");
|
||||||
|
for (const u of srs2.uncertainties) {
|
||||||
|
parts.push(` - ${u}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (srs2.assumptions && srs2.assumptions.length) {
|
||||||
|
parts.push("Assumptions:");
|
||||||
|
for (const a of srs2.assumptions) {
|
||||||
|
parts.push(` - ${a}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (srs2.relationships && srs2.relationships.length) {
|
||||||
|
parts.push("Relationships (internal to fragment):");
|
||||||
|
for (const r of srs2.relationships) {
|
||||||
|
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts.push("");
|
||||||
|
|
||||||
|
// Borderline-control fragment (later)
|
||||||
|
parts.push("=== Fragment (later evidence) ===");
|
||||||
|
parts.push(`Question: ${controlSrs.question}`);
|
||||||
|
if (controlSrs.observations && controlSrs.observations.length) {
|
||||||
|
parts.push("Observations:");
|
||||||
|
for (const o of controlSrs.observations) {
|
||||||
|
parts.push(` - ${o}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (controlSrs.uncertainties && controlSrs.uncertainties.length) {
|
||||||
|
parts.push("Uncertainties:");
|
||||||
|
for (const u of controlSrs.uncertainties) {
|
||||||
|
parts.push(` - ${u}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (controlSrs.assumptions && controlSrs.assumptions.length) {
|
||||||
|
parts.push("Assumptions:");
|
||||||
|
for (const a of controlSrs.assumptions) {
|
||||||
|
parts.push(` - ${a}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (controlSrs.relationships && controlSrs.relationships.length) {
|
||||||
|
parts.push("Relationships (internal to fragment):");
|
||||||
|
for (const r of controlSrs.relationships) {
|
||||||
|
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts.push("");
|
||||||
|
|
||||||
|
// Core question
|
||||||
|
parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,");
|
||||||
|
parts.push("assumption or observation in the earlier fragment?");
|
||||||
|
parts.push("");
|
||||||
|
|
||||||
|
// Return format — same contract as RTO.21 / RTO.22
|
||||||
|
parts.push('Return exactly one JSON object:');
|
||||||
|
parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
|
||||||
|
parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');
|
||||||
|
|
||||||
|
return parts.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Output contract validation (same as RTO.21 / RTO.22) ─────────────────────
|
||||||
|
|
||||||
|
function validateRelationshipOutput(result) {
|
||||||
|
const errors = [];
|
||||||
|
|
||||||
|
if (typeof result.relationshipFound !== "boolean") {
|
||||||
|
errors.push("relationshipFound must be a boolean");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.relationshipFound) {
|
||||||
|
if (result.relationship !== null && result.relationship !== undefined) {
|
||||||
|
errors.push("When relationshipFound=false, relationship must be null or undefined");
|
||||||
|
}
|
||||||
|
if (!Array.isArray(result.remainingQualification)) {
|
||||||
|
errors.push("remainingQualification must be an array when relationshipFound=false");
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
// When true, relationship object is required
|
||||||
|
if (!result.relationship || typeof result.relationship !== "object") {
|
||||||
|
errors.push("When relationshipFound=true, relationship must be an object");
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rel = result.relationship;
|
||||||
|
if (typeof rel.laterEvidence !== "string") {
|
||||||
|
errors.push("relationship.laterEvidence must be a string");
|
||||||
|
}
|
||||||
|
if (typeof rel.earlierItem !== "string") {
|
||||||
|
errors.push("relationship.earlierItem must be a string");
|
||||||
|
}
|
||||||
|
if (typeof rel.bearing !== "string") {
|
||||||
|
errors.push("relationship.bearing must be a string");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Array.isArray(result.remainingQualification)) {
|
||||||
|
errors.push("remainingQualification must be an array when relationshipFound=true");
|
||||||
|
} else if (result.remainingQualification.some((s) => typeof s !== "string")) {
|
||||||
|
errors.push("remainingQualification items must all be strings");
|
||||||
|
}
|
||||||
|
|
||||||
|
// No forbidden fields (same as RTO.21 / RTO.22)
|
||||||
|
const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
|
||||||
|
"recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
|
||||||
|
for (const field of forbidden) {
|
||||||
|
if (field in result && typeof result[field] !== "undefined") {
|
||||||
|
errors.push(`Forbidden field present: ${field}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Prompt-leak check ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function checkPromptLeak(prompt) {
|
||||||
|
const lowerPrompt = prompt.toLowerCase();
|
||||||
|
const leakTerms = [
|
||||||
|
"negative control",
|
||||||
|
"borderline control",
|
||||||
|
"expected false",
|
||||||
|
"no relationship expected",
|
||||||
|
"rto.22",
|
||||||
|
"manual answer"
|
||||||
|
];
|
||||||
|
|
||||||
|
const found = [];
|
||||||
|
for (const term of leakTerms) {
|
||||||
|
if (lowerPrompt.includes(term.toLowerCase())) {
|
||||||
|
found.push(term);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
leaked: found.length > 0,
|
||||||
|
terms: found
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Inspect mode (default: zero model calls) ────────────────────────────────
|
||||||
|
|
||||||
|
async function inspectApparatus() {
|
||||||
|
console.log("=== RTO.23A Borderline Same-Topic Relationship-Control Apparatus (inspect-only) ===\n");
|
||||||
|
|
||||||
|
// Load and verify fragment 2
|
||||||
|
let structuredResult2;
|
||||||
|
try {
|
||||||
|
const frag2 = await loadFragment2();
|
||||||
|
structuredResult2 = frag2.structuredResult;
|
||||||
|
} catch (e) {
|
||||||
|
console.log("ERROR: Cannot load Fragment 2:", e.message);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Borderline-control integrity check
|
||||||
|
const controlCheck = verifyBorderlineControlClean();
|
||||||
|
console.log("--- Borderline-control fixture integrity ---");
|
||||||
|
console.log(`Same competitor/market context present: ${controlCheck.sameCompetitorContext ? "YES" : "NO"}`);
|
||||||
|
console.log(`Same conference context present: ${controlCheck.sameConferenceContext ? "YES" : "NO"}`);
|
||||||
|
console.log(`General customer-problem reference present: ${controlCheck.customerProblemRef ? "YES" : "NO"}`);
|
||||||
|
console.log(`Technical architecture present: ${controlCheck.technicalArchitecture ? "YES" : "NO"}`);
|
||||||
|
console.log(`Prototype evidence present: ${controlCheck.prototypeEvidence ? "YES" : "NO"}`);
|
||||||
|
console.log(`Product roadmap present: ${controlCheck.productRoadmap ? "YES" : "NO"}`);
|
||||||
|
console.log(`Launch timing evidence present: ${controlCheck.launchTiming ? "YES" : "NO"}`);
|
||||||
|
console.log(`Development programme present: ${controlCheck.developmentProgramme ? "YES" : "NO"}`);
|
||||||
|
console.log(`New hiring evidence present: ${controlCheck.newHiringEvidence ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
if (!controlCheck.clean) {
|
||||||
|
console.log("\nBORDERLINE_CONTROL_CLEAN:\nNO");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log("\nBORDERLINE_CONTROL_CLEAN:\nYES");
|
||||||
|
|
||||||
|
// Fragment verification
|
||||||
|
const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
|
||||||
|
console.log("\n--- Fragment verification ---");
|
||||||
|
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||||
|
console.log(`Borderline-control fragment supplied: YES`);
|
||||||
|
|
||||||
|
// Build prompt for inspection
|
||||||
|
const prompt = buildRelationshipPrompt(structuredResult2, BORDERLINE_CONTROL_FRAGMENT);
|
||||||
|
|
||||||
|
// Measure character sizes
|
||||||
|
const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
|
||||||
|
const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
|
||||||
|
const controlStructuredText = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2);
|
||||||
|
|
||||||
|
console.log("\n--- Context-size measurement ---");
|
||||||
|
console.log(`instructionCharacterCount: ${instructionPart.length}`);
|
||||||
|
console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
|
||||||
|
console.log(`borderlineControlCharacterCount: ${controlStructuredText.length}`);
|
||||||
|
console.log(`inputCharacterCount: ${prompt.length}`);
|
||||||
|
|
||||||
|
// ── Boundary checks ───────────────────────────────────────────
|
||||||
|
|
||||||
|
console.log("\n--- Input boundary verification ---");
|
||||||
|
|
||||||
|
const fragment3Supplied = prompt.includes("Fragment 3");
|
||||||
|
console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
const rto21PositiveResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
|
||||||
|
prompt.includes("active competing-product development more plausible");
|
||||||
|
console.log(`RTO.21 positive result supplied: ${rto21PositiveResultSupplied ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
|
||||||
|
prompt.includes("whole-case");
|
||||||
|
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
const centralCaseInPrompt = prompt.includes("central case") ||
|
||||||
|
prompt.includes("centralStatement");
|
||||||
|
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
const currentViewInPrompt = prompt.includes("derived view") ||
|
||||||
|
prompt.includes("focused view") || prompt.includes("current focused");
|
||||||
|
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
|
||||||
|
prompt.includes("other fragments") || prompt.includes("additional fragment");
|
||||||
|
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
const turnHistoryInPrompt = prompt.includes("turn history") ||
|
||||||
|
prompt.includes("prior turns") || prompt.includes("conversation history");
|
||||||
|
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
// ── Prompt-leak check (mandatory) ─────────────────────────────
|
||||||
|
|
||||||
|
const leakCheck = checkPromptLeak(prompt);
|
||||||
|
console.log("\n--- Prompt-leak check ---");
|
||||||
|
console.log(`EXPECTED_OUTCOME_LEAKED_TO_MODEL:\n${leakCheck.leaked ? "YES" : "NO"}`);
|
||||||
|
if (leakCheck.leaked) {
|
||||||
|
console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Output contract validation capability ─────────────────────
|
||||||
|
|
||||||
|
const mockProposal = {
|
||||||
|
relationshipFound: true,
|
||||||
|
relationship: {
|
||||||
|
laterEvidence: "test",
|
||||||
|
earlierItem: "test",
|
||||||
|
bearing: "test",
|
||||||
|
},
|
||||||
|
remainingQualification: ["test"],
|
||||||
|
};
|
||||||
|
const mockNoRel = {
|
||||||
|
relationshipFound: false,
|
||||||
|
relationship: null,
|
||||||
|
remainingQualification: [],
|
||||||
|
};
|
||||||
|
const proposalErrors = validateRelationshipOutput(mockProposal);
|
||||||
|
const noRelErrors = validateRelationshipOutput(mockNoRel);
|
||||||
|
console.log("\n--- Output contract (same as RTO.21) ---");
|
||||||
|
console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
|
||||||
|
console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
|
||||||
|
console.log(`relationshipFound=false allowed: YES`);
|
||||||
|
console.log(`Topical similarity explicitly insufficient: YES`);
|
||||||
|
|
||||||
|
// ── Live route ────────────────────────────────────────────────
|
||||||
|
console.log("\n--- Live route ---");
|
||||||
|
console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live");
|
||||||
|
|
||||||
|
// ── Environment check ─────────────────────────────────────────
|
||||||
|
const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
|
||||||
|
const modelConfigured = !!process.env.OLLAMA_MODEL;
|
||||||
|
console.log("\n--- Environment ---");
|
||||||
|
console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`);
|
||||||
|
console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);
|
||||||
|
|
||||||
|
// ── Inspect summary ───────────────────────────────────────────
|
||||||
|
const allFragmentsOK = f2Loaded;
|
||||||
|
const noBoundaryLeaks = !fragment3Supplied && !rto21PositiveResultSupplied &&
|
||||||
|
!wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt &&
|
||||||
|
!otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean && !leakCheck.leaked;
|
||||||
|
|
||||||
|
console.log("\n=== SUMMARY ===");
|
||||||
|
console.log(`Inspect live calls: 0`);
|
||||||
|
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||||
|
console.log(`Borderline-control fragment supplied: YES`);
|
||||||
|
console.log(`Fragment 3 supplied: NO`);
|
||||||
|
console.log(`RTO.21 positive result supplied: NO`);
|
||||||
|
console.log(`RTO.22 negative result supplied: NO`);
|
||||||
|
console.log(`Known relationship supplied: NO`);
|
||||||
|
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
|
||||||
|
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
|
||||||
|
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
|
||||||
|
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||||
|
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
|
||||||
|
console.log(`BORDERLINE_CONTROL_CLEAN:\nYES`);
|
||||||
|
console.log(`--live route exposed: YES`);
|
||||||
|
console.log(`maximum live calls per --live: 1`);
|
||||||
|
console.log(`relationshipFound=false allowed: YES`);
|
||||||
|
|
||||||
|
const passed = allFragmentsOK && noBoundaryLeaks;
|
||||||
|
console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
|
||||||
|
|
||||||
|
if (!passed) {
|
||||||
|
console.log("\nApparatus cannot satisfy boundary without production changes.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { prompt };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Live execution (exactly one model call) ──────────────────────────────────
|
||||||
|
|
||||||
|
async function executeLive() {
|
||||||
|
const baseUrl = process.env.OLLAMA_BASE_URL;
|
||||||
|
if (!baseUrl) {
|
||||||
|
console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||||
|
console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelName = process.env.OLLAMA_MODEL;
|
||||||
|
if (!modelName) {
|
||||||
|
console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load fragment 2 and build prompt
|
||||||
|
const { structuredResult: srs2 } = await loadFragment2();
|
||||||
|
const prompt = buildRelationshipPrompt(srs2, BORDERLINE_CONTROL_FRAGMENT);
|
||||||
|
|
||||||
|
// Prompt-leak check before sending
|
||||||
|
const leakCheck = checkPromptLeak(prompt);
|
||||||
|
if (leakCheck.leaked) {
|
||||||
|
console.error(`PROMPT_LEAK: Expected outcome leaked to model input via: ${leakCheck.terms.join(", ")}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare artifact path for live result
|
||||||
|
await fs.mkdir(resultsDir, { recursive: true });
|
||||||
|
|
||||||
|
const startedAt = Date.now();
|
||||||
|
console.log("Live mode: sending to model...");
|
||||||
|
|
||||||
|
const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
|
||||||
|
const raw = await provider.generateReconstruction(prompt, modelName);
|
||||||
|
const elapsedMs = Date.now() - startedAt;
|
||||||
|
|
||||||
|
console.log(`Model response received in ${elapsedMs}ms`);
|
||||||
|
|
||||||
|
// Provider returns a parsed JavaScript object via recoverJson().
|
||||||
|
const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => {
|
||||||
|
console.error("Provider returned unexpected type (expected parsed object):", typeof raw);
|
||||||
|
process.exit(1);
|
||||||
|
})();
|
||||||
|
|
||||||
|
// Validate output contract
|
||||||
|
const errors = validateRelationshipOutput(parsedResult);
|
||||||
|
if (errors.length > 0) {
|
||||||
|
console.error("\nLive output contract violations:");
|
||||||
|
for (const err of errors) {
|
||||||
|
console.error(` - ${err}`);
|
||||||
|
}
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write artifact
|
||||||
|
const artifactPath = path.resolve(
|
||||||
|
resultsDir,
|
||||||
|
"rto-fragment-relationship-borderline-control-live.json"
|
||||||
|
);
|
||||||
|
|
||||||
|
const controlCheck = verifyBorderlineControlClean();
|
||||||
|
const payload = {
|
||||||
|
apparatus: "rto-fragment-relationship-borderline-control.mjs",
|
||||||
|
experiment: "RTO.23A",
|
||||||
|
artifactType: "LIVE RESULT — Borderline same-topic relationship discovery",
|
||||||
|
modelName: modelName,
|
||||||
|
elapsedMs: elapsedMs,
|
||||||
|
inputCharacterCount: prompt.length,
|
||||||
|
instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
|
||||||
|
fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
|
||||||
|
borderlineControlCharacterCount: JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2).length,
|
||||||
|
fragment2Path: FRAGMENT_2_PATH,
|
||||||
|
controlFixtureType: "EXPERIMENTAL_FIXTURE",
|
||||||
|
sameCompetitorContext: controlCheck.sameCompetitorContext,
|
||||||
|
sameConferenceContext: controlCheck.sameConferenceContext,
|
||||||
|
borderlineControlClean: controlCheck.clean,
|
||||||
|
knownRelationshipSuppliedToModel: false,
|
||||||
|
wholeSituationGraphSupplied: false,
|
||||||
|
centralCaseStatementSupplied: false,
|
||||||
|
turnHistorySupplied: false,
|
||||||
|
proposedRelationship: parsedResult,
|
||||||
|
};
|
||||||
|
|
||||||
|
await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
|
||||||
|
console.log(`\nLive result written to: ${artifactPath}`);
|
||||||
|
console.log("\n=== Proposed Relationship ===");
|
||||||
|
console.log(JSON.stringify(parsedResult, null, 2));
|
||||||
|
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── CLI entry point ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
|
||||||
|
// --live flag for exactly one live model call
|
||||||
|
if (args.includes("--live")) {
|
||||||
|
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
|
||||||
|
console.log("WARNING: This will make exactly ONE live model call.\n");
|
||||||
|
const result = await executeLive();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default: inspect-only, zero model calls
|
||||||
|
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
|
||||||
|
console.log("Run with --live for one live model call.\n");
|
||||||
|
|
||||||
|
const result = await inspectApparatus();
|
||||||
|
if (result === null) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(error instanceof Error ? error.message : String(error));
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user