test(experiment): checkpoint comparison observability

2026-08-18 19:28:30 +01:00
parent 952cb442b5
commit 913dfec507
5 changed files with 464 additions and 22 deletions
@@ -67,9 +67,11 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
  const { getProvider } = await import("../../lib/llm/provider.js");
  const provider = getProvider();
  let modelElapsedMs = null;
+  let providerGenerateCalls = 0;

  const timedProvider = {
    async generateReconstruction(prompt, modelName) {
+      providerGenerateCalls++;
      const startedAt = Date.now();
      const response = await provider.generateReconstruction(prompt, modelName);
      modelElapsedMs = Date.now() - startedAt;
@@ -92,6 +94,7 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
  return {
    modelElapsedMs,
    endToEndElapsedMs,
+    providerGenerateCalls,
    modelName: model,
    userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
    possibleInference:    answerMeaning.possibleInference    ?? null,