confidence-engine/scripts/debug-ollama-direct.mjs

/**
 * Debug script: send raw Ollama requests directly, bypassing the application provider.
 * Tests /api/chat with format:json and captures request payloads + raw responses.
 */
import { mkdirSync, writeFileSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = dirname(fileURLToPath(import.meta.url));

const BASE_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
const TIMESTAMP = new Date().toISOString().replace(/[/:]/g, "-");
const RESULTS_DIR = join(__dirname, "..", "provider-debug-results", TIMESTAMP);

mkdirSync(RESULTS_DIR, { recursive: true });

// ============================================================
// Test cases
// ============================================================

const MODEL_A = "qwen-claude:latest";
const MODEL_B = "qwen3.6:35b-a3b";

function getModelList() {
  // Check which models are available locally (not via Ollama server)
  return { A: MODEL_A, B: MODEL_B };
}

// Test A: Simple text reply to verify model responds normally
const TEST_A = {
  label: "A",
  description: "Plain instruction test — should return CHAT_WORKS",
  system: "You are a normal assistant. Follow the user instruction exactly.",
  user: "Reply with exactly: CHAT_WORKS",
};

// Test B: Explicit JSON schema via format field
const TEST_B = {
  label: "B",
  description: "JSON schema test — should return exact object",
  system: null, // uses messages only with format
  user: 'Return exactly: {"message": "STRUCTURED_OUTPUT_WORKS"}',
};

// Test C: Minimal reconstruction-style schema
const TEST_C = {
  label: "C",
  description: "Minimal reconstruction schema — structured output test",
  system: null,
  user: "Analyse this situation without solving it: Some customers can log in but cannot download invoices. Identify the meaningful difference and ask one useful next question.",
};

const ALL_TESTS = [TEST_A, TEST_B, TEST_C];

// ============================================================
// Helper functions
// ============================================================

async function runChatWithFormat(model, messages, format) {
  const body = { model, messages, stream: false, format };
  const res = await fetch(`${BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(body),
  });

  const rawText = await res.text();
  let parsed = null;
  try { parsed = JSON.parse(rawText); } catch {}

  return {
    status: res.status,
    statusText: res.statusText,
    requestPayload: body,
    rawResponseText: rawText.slice(0, 5000),
    parsedResponse: parsed,
    messageContent: parsed?.message?.content ?? null,
    thinkingLength: (parsed?.message?.thinking || "").length,
    messageContentType: typeof parsed?.message?.content,
    responseField: parsed?.response,
  };
}

async function runGenerate(model, prompt) {
  const body = { model, prompt, stream: false };
  const res = await fetch(`${BASE_URL}/api/generate`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(body),
  });

  const rawText = await res.text();
  let parsed = null;
  try { parsed = JSON.parse(rawText); } catch {}

  return {
    status: res.status,
    requestPayload: body,
    rawResponseText: rawText.slice(0, 5000),
    parsedResponse: parsed,
    responseField: typeof parsed?.response === "string" ? parsed.response : JSON.stringify(parsed),
    responseFirst200: (parsed?.response || "").slice(0, 200),
  };
}

// ============================================================
// Run tests
// ============================================================

const results = {};

for (const model of [MODEL_A, MODEL_B]) {
  console.log(`\n=== Testing model: ${model} ===`);
  results[model] = {};

  // Check if model is available locally
  let available = false;
  try {
    const tagsRes = await fetch(`${BASE_URL}/api/tags`);
    const tagsData = await tagsRes.json();
    available = tagsData.models?.some(m => m.name.includes(model.split(":")[0]));
  } catch (e) {
    console.log(`  Warning: could not check model availability: ${e.message}`);
  }

  if (!available) {
    results[model].availability = "NOT_AVAILABLE_ON_SERVER";
    console.log(`  -> Model ${model} not found on server, skipping`);
    continue;
  }

  console.log(`  -> Model available on server\n`);

  for (const test of ALL_TESTS) {
    const testKey = `test_${test.label}_${model.split(":")[0].replace(/[^a-zA-Z]/g, "_")}`;
    console.log(`  Running Test ${test.label}: ${test.description}`);

    // Chat with format:json
    let chatResult;
    try {
      const messages = [];
      if (test.system) {
        messages.push({ role: "system", content: test.system });
      }
      messages.push({ role: "user", content: test.user });

      chatResult = await runChatWithFormat(model, messages, "json");
      
      // Try to extract JSON from message.content
      let extractedJson = null;
      if (typeof chatResult.messageContent === "string") {
        try {
          extractedJson = JSON.parse(chatResult.messageContent);
        } catch {}
      }

      results[model][testKey] = {
        testDescription: test.description,
        endpoint: "/api/chat",
        format: "json",
        hasSystemMessage: !!test.system,
        httpStatus: chatResult.status,
        messageContentType: chatResult.messageContentType,
        messageContentLength: chatResult.messageContent?.length || 0,
        thinkingPresent: chatResult.thinkingLength > 0,
        parsedContentKeys: extractedJson ? Object.keys(extractedJson) : null,
        // If content looks like a status acknowledgment
        looksLikeStatusAck: typeof chatResult.messageContent === "string" && 
          (chatResult.messageContent.includes('"status"') || chatResult.messageContent.includes('"state"')),
        rawPreview: chatResult.messageContent?.slice(0, 300) ?? "(none)",
      };

      const status = extractedJson ? "JSON_OK" : (chatResult.messageContent ? "TEXT_RESPONSE" : "EMPTY");
      console.log(`    -> ${status} (HTTP ${chatResult.status}, content type: ${chatResult.messageContentType})`);
      if (extractedJson) {
        console.log(`       JSON keys: ${Object.keys(extractedJson).join(", ")}`);
      } else if (chatResult.messageContent) {
        console.log(`       Content preview: ${(typeof chatResult.messageContent === "string" ? chatResult.messageContent : String(chatResult.messageContent)).slice(0, 150)}...`);
      }

    } catch (e) {
      results[model][testKey] = { error: e.message };
      console.log(`    -> ERROR: ${e.message}`);
    }

    // Generate (fallback test)
    let generateResult;
    try {
      const generatePrompt = test.system ? `${test.system}\n\n${test.user}` : test.user;
      generateResult = await runGenerate(model, generatePrompt);

      results[model][`${testKey}_generate`] = {
        endpoint: "/api/generate",
        httpStatus: generateResult.status,
        responseFirst200: generateResult.responseFirst200,
        responseLooksLikeStructuredJSON: generateResult.responseField?.trim().startsWith("{"),
        rawPreview: generateResult.responseFirst200,
      };

      const isJson = generateResult.responseField?.trim().startsWith("{") ? "JSON_START" : "NOT_JSON";
      console.log(`    -> ${isJson} (HTTP ${generateResult.status})`);

    } catch (e) {
      results[model][`${testKey}_generate`] = { error: e.message };
      console.log(`    -> GENERATE ERROR: ${e.message}`);
    }

    console.log();
  }
}

// ============================================================
// Save results
// ============================================================

const saveFile = join(RESULTS_DIR, "debug-results.json");
writeFileSync(saveFile, JSON.stringify(results, null, 2));
console.log(`\nResults saved to: ${saveFile}`);