test(experiment): checkpoint focused vs global comparison apparatus
This commit is contained in:
@@ -64,7 +64,25 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
|
||||
"../../lib/graph/orchestrator.js"
|
||||
);
|
||||
|
||||
const result = await updateCase({ situationGraph: graph, previousQuestion, answer });
|
||||
const { getProvider } = await import("../../lib/llm/provider.js");
|
||||
const provider = getProvider();
|
||||
let modelElapsedMs = null;
|
||||
|
||||
const timedProvider = {
|
||||
async generateReconstruction(prompt, modelName) {
|
||||
const startedAt = Date.now();
|
||||
const response = await provider.generateReconstruction(prompt, modelName);
|
||||
modelElapsedMs = Date.now() - startedAt;
|
||||
return response;
|
||||
},
|
||||
};
|
||||
|
||||
const endToEndStartedAt = Date.now();
|
||||
const result = await updateCase(
|
||||
{ situationGraph: graph, previousQuestion, answer },
|
||||
{ provider: timedProvider },
|
||||
);
|
||||
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
||||
|
||||
// Extract the standard checkpoints
|
||||
const answerMeaning = result?.proposal?.answerMeaning || {};
|
||||
@@ -72,6 +90,9 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
|
||||
const supportedCat = _deriveCategory(answerMeaning.userSupportedMeaning || "");
|
||||
|
||||
return {
|
||||
modelElapsedMs,
|
||||
endToEndElapsedMs,
|
||||
modelName: model,
|
||||
userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
|
||||
possibleInference: answerMeaning.possibleInference ?? null,
|
||||
rawAnswerCategory: rawAnswerCat,
|
||||
|
||||
Reference in New Issue
Block a user