test(experiment): checkpoint focused vs global comparison apparatus

2026-08-18 18:34:08 +01:00
parent 2f6c90b027
commit 952cb442b5
2 changed files with 277 additions and 1 deletions
@@ -64,7 +64,25 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
    "../../lib/graph/orchestrator.js"
  );

-  const result = await updateCase({ situationGraph: graph, previousQuestion, answer });
+  const { getProvider } = await import("../../lib/llm/provider.js");
+  const provider = getProvider();
+  let modelElapsedMs = null;
+
+  const timedProvider = {
+    async generateReconstruction(prompt, modelName) {
+      const startedAt = Date.now();
+      const response = await provider.generateReconstruction(prompt, modelName);
+      modelElapsedMs = Date.now() - startedAt;
+      return response;
+    },
+  };
+
+  const endToEndStartedAt = Date.now();
+  const result = await updateCase(
+    { situationGraph: graph, previousQuestion, answer },
+    { provider: timedProvider },
+  );
+  const endToEndElapsedMs = Date.now() - endToEndStartedAt;

  // Extract the standard checkpoints
  const answerMeaning  = result?.proposal?.answerMeaning || {};
@@ -72,6 +90,9 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
  const supportedCat   = _deriveCategory(answerMeaning.userSupportedMeaning || "");

  return {
+    modelElapsedMs,
+    endToEndElapsedMs,
+    modelName: model,
    userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
    possibleInference:    answerMeaning.possibleInference    ?? null,
    rawAnswerCategory:    rawAnswerCat,