test(experiment): checkpoint comparison observability

scripts/experimental/rto-focused-vs-global-comparison.mjs

@@ -4,13 +4,17 @@ import dotenv from "dotenv";
 
 import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" };
 import { buildGraphUpdatePrompt } from "../../lib/graph/prompt-builder.js";
-import { getProvider } from "../../lib/llm/provider.js";
 import { assertConfig } from "../../lib/config.js";
 
 import { createRequire } from "module";
 const require = createRequire(import.meta.url);
 const { runLiveExperiment } = require("../../tests/graph/live-update-experiment-helper.cjs");
 
+// Non-production diagnostics: an independent request captures raw-response
+// text and Ollama completion metadata (done, done_reason, prompt_eval_count,
+// eval_count) before delegating JSON recovery to the inline helper.
+import { runFocusedDiagnostic } from "./focus-diagnostics-helper.mjs";
+
 dotenv.config({ path: ".env.local" });
 
 const TARGET_NODE_ID = "nxmeiab";
@@ -135,29 +139,76 @@ function buildGlobalPlan() {
 }
 
 async function runFocused() {
-  const { modelName } = validateEnvironment();
-  const provider = getProvider();
+  const { baseUrl, modelName } = validateEnvironment();
   const plan = buildFocusedPlan();
 
-  const startedAt = Date.now();
-  const raw = await provider.generateReconstruction(plan.prompt, modelName);
-  const modelElapsedMs = Date.now() - startedAt;
+  // ---- non-production diagnostics wrapper ----
+  // Independent request captures raw response + completion metadata.
+  // The diagnostics helper mirrors the production JSON-recovery logic inline
+  // (since recoverJson is not exported) and delegates parse back to that
+  // implementation. The comparison runner only observes result/error objects.
+  const diagResult = await runFocusedDiagnostic({ baseUrl, modelName, prompt: plan.prompt });
 
-  const artifact = {
-    path: "focused",
-    targetNodeId: TARGET_NODE_ID,
-    modelName,
-    modelElapsedMs,
-    inputCharacterCount: plan.inputCharacterCount,
-    structuredResult: raw,
-  };
+  try {
+    if (diagResult.error) {
+      throw diagResult.error;
+    }
+    // Successful parse via production helper — proceed to standard artifact
+    const raw = diagResult.parsedResult;
 
-  const artifactPath = path.resolve(
-    "tests/experimental/artifacts/rto-focused-vs-global-focused.json",
-  );
-  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
-  await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
-  console.log(JSON.stringify({ artifactPath, modelElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2));
+    const artifact = {
+      path: "focused",
+      targetNodeId: TARGET_NODE_ID,
+      modelName,
+      modelElapsedMs: diagResult.diagnostics.modelElapsedMs,
+      inputCharacterCount: plan.inputCharacterCount,
+      structuredResult: raw,
+    };
+
+    const artifactPath = path.resolve(
+      "tests/experimental/artifacts/rto-focused-vs-global-focused.json",
+    );
+    await fs.mkdir(path.dirname(artifactPath), { recursive: true });
+    await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
+    console.log(JSON.stringify({ artifactPath, modelElapsedMs: diagResult.diagnostics.modelElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2));
+    return;
+  } catch (err) {
+    const focusedDiagnostics = err.focusedDiagnostics ?? null;
+
+    if (!focusedDiagnostics) {
+      // Should not happen — every DiagnosticError carries diagnostics.
+      console.error("Unexpected: no diagnostics captured before parse failure.");
+      process.exit(1);
+    }
+
+    const errorArtifact = {
+      path: "focused",
+      modelName: focusedDiagnostics.apiUsed === "/api/chat" ? modelName : modelName,
+      inputCharacterCount: plan.inputCharacterCount,
+      rawResponseText: focusedDiagnostics.rawResponseText?.slice(0, 4096),
+      rawResponseCharacterCount: focusedDiagnostics.rawResponseCharacterCount,
+      done: focusedDiagnostics.done,
+      doneReason: focusedDiagnostics.doneReason,
+      promptEvalCount: focusedDiagnostics.promptEvalCount,
+      evalCount: focusedDiagnostics.evalCount,
+      errorType: err.name,
+      errorMessage: err.message.slice(0, 4096),
+    };
+
+    const artifactPath = path.resolve(
+      "tests/experimental/artifacts/rto-focused-vs-global-focused-failure.json",
+    );
+    await fs.mkdir(path.dirname(artifactPath), { recursive: true });
+    await fs.writeFile(artifactPath, JSON.stringify(errorArtifact, null, 2));
+
+    console.log(JSON.stringify({
+      artifactPath,
+      errorType: err.name,
+      rawResponseCharacterCount: focusedDiagnostics.rawResponseCharacterCount,
+      done: focusedDiagnostics.done,
+      doneReason: focusedDiagnostics.doneReason,
+    }, null, 2));
+  }
 }
 
 async function runGlobal() {
@@ -177,6 +228,8 @@ async function runGlobal() {
     targetNodeId: TARGET_NODE_ID,
     modelName,
     modelElapsedMs: result.modelElapsedMs,
+    providerGenerateCalls: result.providerGenerateCalls ?? 0,
+    providerTimingCaptured: typeof result.modelElapsedMs === "number",
     endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs,
     inputCharacterCount: plan.inputCharacterCount,
     graphUpdateResultSummary: {
@@ -213,7 +266,7 @@ function printStaticValidation() {
     fixedQuestion: FIXED_QUESTION,
     fixedAnswer: FIXED_ANSWER,
     focused: {
-      resolvesTo: "scripts/experimental/rto-focused-answer-deconstruction.mjs-equivalent focused provider path",
+      resolvesTo: "scripts/experimental/rto-focused-vs-global-comparison-equivalent focused provider path",
       wholeGraphSupplied: false,
       inputCharacterCount: focused.inputCharacterCount,
       modelTimingBoundary: focused.modelTimingBoundary,
@@ -252,4 +305,4 @@ if (mode === "--focused") {
 } else {
   console.error(`Unknown mode: ${mode}`);
   process.exit(1);
-}
+}