Merge pull request 'feature/cl8-instance-health' (#25) from feature/cl8-instance-health into master

Reviewed-on: http://gitea.lan:3000/robbond/pleskSaas/pulls/25

app/api/cron/health/route.ts

@@ -0,0 +1,38 @@
+import { NextResponse } from "next/server";
+
+import { env } from "@/lib/env";
+import { runHealthCheckTick } from "@/lib/plesk/health";
+import { createSupabaseAdminClient } from "@/lib/supabase/admin";
+
+export async function POST(req: Request) {
+  const startedAt = Date.now();
+
+  try {
+    const secret = req.headers.get("x-cron-secret");
+    if (!env.cronSecret || !secret || secret !== env.cronSecret) {
+      return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+    }
+
+    const supabase = createSupabaseAdminClient() as any;
+    const summary = await runHealthCheckTick(supabase);
+
+    return NextResponse.json({
+      ...summary,
+      duration_ms: Date.now() - startedAt,
+    });
+  } catch (error) {
+    console.error("[health cron] failed", error);
+
+    return NextResponse.json(
+      {
+        error: "Health check job failed",
+        checked: 0,
+        ok: 0,
+        degraded: 0,
+        failed: 0,
+        duration_ms: Date.now() - startedAt,
+      },
+      { status: 500 },
+    );
+  }
+}

app/api/plesk/instances/[id]/health/route.ts

@@ -0,0 +1,73 @@
+import { NextResponse } from "next/server";
+
+import { assertAgencyAdmin, getRouteAgencyContextOrThrow } from "@/lib/agency";
+import { assertRateLimit } from "@/lib/api/rate-limit";
+import { assertSameOrigin } from "@/lib/api/security";
+import { runInstanceHealthCheck } from "@/lib/plesk/health";
+import { createSupabaseRouteClient } from "@/lib/supabase/route";
+
+export async function POST(
+  req: Request,
+  { params }: { params: { id: string } },
+) {
+  try {
+    assertSameOrigin(req);
+
+    const context = await getRouteAgencyContextOrThrow();
+    assertAgencyAdmin(context);
+    assertRateLimit(`plesk-health-check:${context.userId}`, 20, 60_000);
+
+    const supabase = createSupabaseRouteClient() as any;
+
+    const { data: instance, error: instanceError } = await supabase
+      .from("plesk_instances")
+      .select(
+        "id, agency_id, base_url, auth_type, encrypted_secret, health_enabled, health_check_frequency_minutes, health_next_check_at, health_lock_until",
+      )
+      .eq("id", params.id)
+      .eq("agency_id", context.agencyId)
+      .single();
+
+    if (instanceError || !instance) {
+      throw new Error("Plesk instance not found");
+    }
+
+    const result = await runInstanceHealthCheck(supabase, instance, {
+      actorUserId: context.userId,
+      force: true,
+      source: "manual",
+    });
+
+    if (!result.checked) {
+      return NextResponse.json(
+        { error: "Health check is already running for this instance" },
+        { status: 409 },
+      );
+    }
+
+    const { data: updated } = await supabase
+      .from("plesk_instances")
+      .select(
+        "id, health_enabled, health_status, health_last_checked_at, health_last_ok_at, health_last_error, health_last_latency_ms, health_check_frequency_minutes, health_next_check_at",
+      )
+      .eq("id", params.id)
+      .eq("agency_id", context.agencyId)
+      .single();
+
+    return NextResponse.json({
+      ok: true,
+      result,
+      instance: updated,
+    });
+  } catch (error) {
+    return NextResponse.json(
+      {
+        error:
+          error instanceof Error
+            ? error.message
+            : "Health check execution failed",
+      },
+      { status: 400 },
+    );
+  }
+}

app/dashboard/page.tsx

@@ -17,6 +17,16 @@ type ActionLogSummary = Pick<
   Database["public"]["Tables"]["actions_log"]["Row"],
   "target_id" | "status" | "action" | "created_at" | "error_message"
 >;
+type HealthLogSummary = Pick<
+  Database["public"]["Tables"]["actions_log"]["Row"],
+  | "id"
+  | "target_id"
+  | "status"
+  | "action"
+  | "created_at"
+  | "error_message"
+  | "metadata"
+>;
 
 function normalizeDomainLike(value: string) {
   return value.trim().toLowerCase().replace(/\.$/, "");
@@ -49,6 +59,7 @@ export default async function DashboardPage() {
     { data: domains },
     { data: billing },
     { data: autoSyncLogs },
+    { data: healthLogs },
   ] = (await Promise.all([
     supabase
       .from("agencies")
@@ -58,7 +69,7 @@ export default async function DashboardPage() {
     supabase
       .from("plesk_instances")
       .select(
-        "id, name, base_url, auth_type, status, error_message, last_connected_at, last_sync_at, last_sync_status, last_sync_error, last_sync_subscriptions, last_sync_domains, auto_sync_enabled, auto_sync_frequency_minutes, last_auto_sync_at, next_auto_sync_at, auto_sync_lock_until",
+        "id, name, base_url, auth_type, status, error_message, last_connected_at, last_sync_at, last_sync_status, last_sync_error, last_sync_subscriptions, last_sync_domains, auto_sync_enabled, auto_sync_frequency_minutes, last_auto_sync_at, next_auto_sync_at, auto_sync_lock_until, health_enabled, health_status, health_last_checked_at, health_last_ok_at, health_last_error, health_last_latency_ms, health_check_frequency_minutes, health_next_check_at, health_lock_until",
       )
       .eq("agency_id", context.agencyId)
       .order("created_at", { ascending: false }),
@@ -89,6 +100,21 @@ export default async function DashboardPage() {
       .eq("action", "plesk_sync_auto")
       .order("created_at", { ascending: false })
       .limit(200),
+    supabase
+      .from("actions_log")
+      .select(
+        "id, target_id, status, action, created_at, error_message, metadata",
+      )
+      .eq("agency_id", context.agencyId)
+      .eq("target_type", "plesk_instance")
+      .in("action", [
+        "health_check_started",
+        "health_check_ok",
+        "health_check_degraded",
+        "health_check_failed",
+      ])
+      .order("created_at", { ascending: false })
+      .limit(400),
   ])) as [
     { data: AgencySummary | null },
     {
@@ -110,6 +136,15 @@ export default async function DashboardPage() {
         last_auto_sync_at: string | null;
         next_auto_sync_at: string | null;
         auto_sync_lock_until: string | null;
+        health_enabled: boolean;
+        health_status: "ok" | "degraded" | "down" | "unknown";
+        health_last_checked_at: string | null;
+        health_last_ok_at: string | null;
+        health_last_error: string | null;
+        health_last_latency_ms: number | null;
+        health_check_frequency_minutes: number;
+        health_next_check_at: string | null;
+        health_lock_until: string | null;
       }> | null;
       error: { message: string } | null;
     },
@@ -138,6 +173,7 @@ export default async function DashboardPage() {
     },
     { data: BillingSummary | null },
     { data: ActionLogSummary[] | null },
+    { data: HealthLogSummary[] | null },
   ];
 
   const domainInstanceIds = Array.from(
@@ -230,6 +266,33 @@ export default async function DashboardPage() {
     });
   }
 
+  const healthEventsByInstance = new Map<
+    string,
+    Array<{
+      id: string;
+      action: string;
+      status: string;
+      created_at: string;
+      error_message: string | null;
+      metadata: unknown;
+    }>
+  >();
+
+  for (const log of healthLogs ?? []) {
+    if (!log.target_id) continue;
+    const existing = healthEventsByInstance.get(log.target_id) ?? [];
+    if (existing.length >= 10) continue;
+    existing.push({
+      id: log.id,
+      action: log.action,
+      status: log.status,
+      created_at: log.created_at,
+      error_message: log.error_message,
+      metadata: log.metadata,
+    });
+    healthEventsByInstance.set(log.target_id, existing);
+  }
+
   return (
     <main className="mx-auto flex w-full max-w-6xl flex-col gap-6 px-6 py-8">
       <header className="flex items-center justify-between">
@@ -312,6 +375,9 @@ export default async function DashboardPage() {
           };
         })}
         autoSyncByInstance={Object.fromEntries(autoSyncByInstance.entries())}
+        healthEventsByInstance={Object.fromEntries(
+          healthEventsByInstance.entries(),
+        )}
       />
     </main>
   );

components/dashboard/dashboard-controls.tsx

@@ -23,6 +23,15 @@ type Instance = {
   last_auto_sync_at: string | null;
   next_auto_sync_at: string | null;
   auto_sync_lock_until: string | null;
+  health_enabled: boolean;
+  health_status: "ok" | "degraded" | "down" | "unknown";
+  health_last_checked_at: string | null;
+  health_last_ok_at: string | null;
+  health_last_error: string | null;
+  health_last_latency_ms: number | null;
+  health_check_frequency_minutes: number;
+  health_next_check_at: string | null;
+  health_lock_until: string | null;
 };
 
 type Subscription = {
@@ -54,8 +63,44 @@ type Props = {
     string,
     { status: string; created_at: string; error_message: string | null }
   >;
+  healthEventsByInstance: Record<
+    string,
+    Array<{
+      id: string;
+      action: string;
+      status: string;
+      created_at: string;
+      error_message: string | null;
+      metadata: unknown;
+    }>
+  >;
 };
 
+function getHealthBadge(status: string | null) {
+  const normalized = (status ?? "unknown").toLowerCase();
+
+  if (normalized === "ok") {
+    return { label: "OK", className: "bg-emerald-100 text-emerald-700" };
+  }
+
+  if (normalized === "degraded") {
+    return { label: "Degraded", className: "bg-amber-100 text-amber-700" };
+  }
+
+  if (normalized === "down") {
+    return { label: "Down", className: "bg-rose-100 text-rose-700" };
+  }
+
+  return { label: "Unknown", className: "bg-slate-100 text-slate-600" };
+}
+
+function mapHealthResult(action: string) {
+  if (action === "health_check_ok") return "ok";
+  if (action === "health_check_degraded") return "degraded";
+  if (action === "health_check_failed") return "failed";
+  return "started";
+}
+
 function getDomainStatusBadge(status: string | null) {
   const normalized = (status ?? "unknown").toLowerCase();
 
@@ -120,6 +165,7 @@ export function DashboardControls({
   subscriptions,
   domains,
   autoSyncByInstance,
+  healthEventsByInstance,
 }: Props) {
   const autoSyncFrequencyOptions = [15, 30, 60, 180, 360, 1440];
   const [instanceName, setInstanceName] = useState("");
@@ -245,6 +291,22 @@ export function DashboardControls({
     }
   }
 
+  async function onRunHealthCheck(instanceId: string) {
+    setLoading(true);
+    setMessage(null);
+    try {
+      await runRequest(`/api/plesk/instances/${instanceId}/health`);
+      setMessage("Health check completed.");
+      window.location.reload();
+    } catch (error) {
+      setMessage(
+        error instanceof Error ? error.message : "Health check failed",
+      );
+    } finally {
+      setLoading(false);
+    }
+  }
+
   async function onUpdateAutoSync(
     instanceId: string,
     input: { auto_sync_enabled: boolean; auto_sync_frequency_minutes: number },
@@ -442,12 +504,17 @@ export function DashboardControls({
                   key={instance.id}
                   className="rounded border border-slate-200 p-3"
                 >
+                  {(() => {
+                    const health = getHealthBadge(instance.health_status);
+
+                    return (
                       <div className="flex items-start justify-between gap-2">
                         <div>
                           {autoSyncByInstance[instance.id] ? (
                             <span
                               className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${
-                            autoSyncByInstance[instance.id].status === "success"
+                                autoSyncByInstance[instance.id].status ===
+                                "success"
                                   ? "bg-emerald-100 text-emerald-700"
                                   : "bg-rose-100 text-rose-700"
                               }`}
@@ -463,10 +530,32 @@ export function DashboardControls({
                           </p>
                           <p className="mt-1 text-xs text-slate-600">
                             Status:{" "}
-                        <span className="font-medium">{instance.status}</span>
+                            <span className="font-medium">
+                              {instance.status}
+                            </span>
                           </p>
                           <p className="text-xs text-slate-500">
-                        Subscriptions: {instance.last_sync_subscriptions ?? 0}
+                            Health:{" "}
+                            <span
+                              title={instance.health_last_error ?? undefined}
+                              className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${health.className}`}
+                            >
+                              {health.label}
+                            </span>
+                          </p>
+                          <p className="text-xs text-slate-500">
+                            Health last checked:{" "}
+                            {formatDateTime(instance.health_last_checked_at)}
+                          </p>
+                          <p className="text-xs text-slate-500">
+                            Health latency:{" "}
+                            {instance.health_last_latency_ms != null
+                              ? `${instance.health_last_latency_ms} ms`
+                              : "—"}
+                          </p>
+                          <p className="text-xs text-slate-500">
+                            Subscriptions:{" "}
+                            {instance.last_sync_subscriptions ?? 0}
                           </p>
                           <p className="text-xs text-slate-500">
                             Domains: {instance.last_sync_domains ?? 0}
@@ -487,7 +576,8 @@ export function DashboardControls({
                             </span>
                           </p>
                           <p className="text-xs text-slate-500">
-                        Last sync time: {formatDateTime(instance.last_sync_at)}
+                            Last sync time:{" "}
+                            {formatDateTime(instance.last_sync_at)}
                           </p>
                           <p className="text-xs text-slate-500">
                             Last connected:{" "}
@@ -497,7 +587,9 @@ export function DashboardControls({
                           </p>
                           <p className="text-xs text-slate-500">
                             Auto-sync:{" "}
-                        {instance.auto_sync_enabled ? "Enabled" : "Disabled"}
+                            {instance.auto_sync_enabled
+                              ? "Enabled"
+                              : "Disabled"}
                           </p>
                           <p className="text-xs text-slate-500">
                             Auto-sync frequency:{" "}
@@ -543,6 +635,14 @@ export function DashboardControls({
                               {autoSyncByInstance[instance.id].error_message}
                             </p>
                           ) : null}
+                          {instance.health_last_error ? (
+                            <p
+                              className="mt-1 text-xs text-rose-600"
+                              title={instance.health_last_error}
+                            >
+                              Health error: {instance.health_last_error}
+                            </p>
+                          ) : null}
                         </div>
                         <div className="flex flex-col gap-2">
                           <div className="flex items-center gap-2">
@@ -569,7 +669,9 @@ export function DashboardControls({
                             onChange={(e: ChangeEvent<HTMLSelectElement>) =>
                               onUpdateAutoSync(instance.id, {
                                 auto_sync_enabled: instance.auto_sync_enabled,
-                            auto_sync_frequency_minutes: Number(e.target.value),
+                                auto_sync_frequency_minutes: Number(
+                                  e.target.value,
+                                ),
                               })
                             }
                             disabled={loading}
@@ -588,6 +690,13 @@ export function DashboardControls({
                           >
                             Test Connection
                           </button>
+                          <button
+                            onClick={() => onRunHealthCheck(instance.id)}
+                            disabled={loading}
+                            className="rounded border px-3 py-1.5 text-xs"
+                          >
+                            Run Health Check
+                          </button>
                           <button
                             onClick={() => onSync(instance.id)}
                             disabled={loading}
@@ -597,6 +706,108 @@ export function DashboardControls({
                           </button>
                         </div>
                       </div>
+                    );
+                  })()}
+
+                  <div className="mt-3 rounded border border-slate-100 bg-slate-50 p-3">
+                    <p className="text-xs font-semibold uppercase tracking-wide text-slate-600">
+                      Health Panel
+                    </p>
+                    <div className="mt-2 grid gap-1 text-xs text-slate-600 sm:grid-cols-2">
+                      <p>
+                        Current Status:{" "}
+                        <span className="font-medium">
+                          {getHealthBadge(instance.health_status).label}
+                        </span>
+                      </p>
+                      <p>
+                        Last Checked:{" "}
+                        <span className="font-medium">
+                          {formatDateTime(instance.health_last_checked_at)}
+                        </span>
+                      </p>
+                      <p>
+                        Last Successful Check:{" "}
+                        <span className="font-medium">
+                          {formatDateTime(instance.health_last_ok_at)}
+                        </span>
+                      </p>
+                      <p>
+                        Latency:{" "}
+                        <span className="font-medium">
+                          {instance.health_last_latency_ms != null
+                            ? `${instance.health_last_latency_ms} ms`
+                            : "—"}
+                        </span>
+                      </p>
+                      <p className="sm:col-span-2">
+                        Last Error:{" "}
+                        <span className="font-medium">
+                          {instance.health_last_error ?? "—"}
+                        </span>
+                      </p>
+                    </div>
+
+                    <div className="mt-3 overflow-x-auto">
+                      <table className="min-w-full text-left text-xs">
+                        <thead>
+                          <tr className="border-b border-slate-200 text-slate-500">
+                            <th className="px-2 py-1">Time</th>
+                            <th className="px-2 py-1">Event</th>
+                            <th className="px-2 py-1">Result</th>
+                            <th className="px-2 py-1">Message</th>
+                          </tr>
+                        </thead>
+                        <tbody>
+                          {(healthEventsByInstance[instance.id] ?? [])
+                            .length === 0 ? (
+                            <tr>
+                              <td
+                                className="px-2 py-2 text-slate-500"
+                                colSpan={4}
+                              >
+                                No health events yet.
+                              </td>
+                            </tr>
+                          ) : (
+                            (healthEventsByInstance[instance.id] ?? []).map(
+                              (event) => (
+                                <tr
+                                  key={event.id}
+                                  className="border-b border-slate-100 align-top"
+                                >
+                                  <td className="px-2 py-1">
+                                    {formatDateTime(event.created_at)}
+                                  </td>
+                                  <td className="px-2 py-1">{event.action}</td>
+                                  <td className="px-2 py-1">
+                                    {mapHealthResult(event.action)}
+                                  </td>
+                                  <td className="px-2 py-1">
+                                    {event.error_message ?? "—"}
+                                    {event.metadata ? (
+                                      <details>
+                                        <summary className="cursor-pointer text-[11px] text-brand-700">
+                                          metadata
+                                        </summary>
+                                        <pre className="mt-1 max-w-[440px] overflow-auto rounded bg-white p-2 text-[10px] text-slate-700">
+                                          {JSON.stringify(
+                                            event.metadata,
+                                            null,
+                                            2,
+                                          )}
+                                        </pre>
+                                      </details>
+                                    ) : null}
+                                  </td>
+                                </tr>
+                              ),
+                            )
+                          )}
+                        </tbody>
+                      </table>
+                    </div>
+                  </div>
                 </div>
               ))
             )}

lib/plesk/health.ts

@@ -0,0 +1,345 @@
+import { randomUUID } from "crypto";
+
+import { testPleskConnection } from "@/lib/plesk/client";
+
+type SupabaseLike = any;
+
+type HealthStatus = "ok" | "degraded" | "down" | "unknown";
+
+type HealthInstance = {
+  id: string;
+  agency_id: string;
+  base_url: string;
+  auth_type: "api_key" | "basic";
+  encrypted_secret: string;
+  health_enabled: boolean;
+  health_check_frequency_minutes: number | null;
+  health_next_check_at: string | null;
+  health_lock_until: string | null;
+};
+
+type HealthCheckResult = {
+  status: Exclude<HealthStatus, "unknown">;
+  latency_ms: number;
+  error_message: string | null;
+};
+
+type RunSingleHealthCheckResult = {
+  checked: boolean;
+  status?: Exclude<HealthStatus, "unknown">;
+  latency_ms?: number;
+  error_message?: string | null;
+};
+
+export type HealthTickSummary = {
+  checked: number;
+  ok: number;
+  degraded: number;
+  failed: number;
+  duration_ms: number;
+};
+
+const MAX_INSTANCES_PER_TICK = 10;
+const MAX_CONCURRENCY = 3;
+const LOCK_LEASE_SECONDS = 5 * 60;
+const HEALTH_TIMEOUT_MS = 15_000;
+const DEGRADED_LATENCY_THRESHOLD_MS = 2_500;
+
+function isDue(instance: HealthInstance, now: number) {
+  if (!instance.health_next_check_at) return true;
+  const dueAt = new Date(instance.health_next_check_at).getTime();
+  return Number.isFinite(dueAt) && dueAt <= now;
+}
+
+function isUnlocked(instance: HealthInstance, now: number) {
+  if (!instance.health_lock_until) return true;
+  const lockUntil = new Date(instance.health_lock_until).getTime();
+  return !Number.isFinite(lockUntil) || lockUntil <= now;
+}
+
+async function insertActionLogBestEffort(
+  supabase: SupabaseLike,
+  payload: Record<string, unknown>,
+) {
+  try {
+    const { error } = await supabase.from("actions_log").insert(payload);
+    if (error) {
+      console.error("[health] actions_log insert failed", error.message);
+    }
+  } catch (error) {
+    console.error("[health] actions_log insert threw", error);
+  }
+}
+
+async function withTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+): Promise<T> {
+  let timeout: ReturnType<typeof setTimeout> | null = null;
+
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => {
+      reject(new Error("Health check timed out"));
+    }, timeoutMs);
+  });
+
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
+
+async function checkPleskInstanceHealth(
+  instance: Pick<HealthInstance, "base_url" | "auth_type" | "encrypted_secret">,
+): Promise<HealthCheckResult> {
+  const startedAt = Date.now();
+
+  try {
+    await withTimeout(
+      testPleskConnection({
+        baseUrl: instance.base_url,
+        authType: instance.auth_type,
+        encryptedSecret: instance.encrypted_secret,
+      }),
+      HEALTH_TIMEOUT_MS,
+    );
+
+    const latencyMs = Date.now() - startedAt;
+    const status: "ok" | "degraded" =
+      latencyMs > DEGRADED_LATENCY_THRESHOLD_MS ? "degraded" : "ok";
+
+    return {
+      status,
+      latency_ms: latencyMs,
+      error_message: null,
+    };
+  } catch (error) {
+    return {
+      status: "down",
+      latency_ms: Date.now() - startedAt,
+      error_message:
+        error instanceof Error
+          ? error.message.slice(0, 1000)
+          : "health_check_failed",
+    };
+  }
+}
+
+export async function runInstanceHealthCheck(
+  supabase: SupabaseLike,
+  instance: HealthInstance,
+  input?: {
+    actorUserId?: string | null;
+    force?: boolean;
+    source?: "cron" | "manual";
+  },
+): Promise<RunSingleHealthCheckResult> {
+  const lockToken = randomUUID();
+  const actorUserId = input?.actorUserId ?? null;
+  const source = input?.source ?? "cron";
+
+  const { data: lockAcquired, error: lockError } = await supabase.rpc(
+    "acquire_instance_health_lock",
+    {
+      p_instance_id: instance.id,
+      p_lock_token: lockToken,
+      p_ttl_seconds: LOCK_LEASE_SECONDS,
+      p_force: Boolean(input?.force),
+    },
+  );
+
+  if (lockError || !lockAcquired) {
+    return { checked: false };
+  }
+
+  const startedAt = Date.now();
+  const nowIso = new Date().toISOString();
+  const frequencyMinutes = Math.max(
+    instance.health_check_frequency_minutes ?? 30,
+    5,
+  );
+  const nextCheckAtIso = new Date(
+    Date.now() + frequencyMinutes * 60 * 1000,
+  ).toISOString();
+
+  await insertActionLogBestEffort(supabase, {
+    agency_id: instance.agency_id,
+    actor_user_id: actorUserId,
+    target_type: "plesk_instance",
+    target_id: instance.id,
+    action: "health_check_started",
+    status: "pending",
+    metadata: {
+      instance_id: instance.id,
+      lock_token: lockToken,
+      source,
+    },
+  });
+
+  try {
+    const result = await checkPleskInstanceHealth(instance);
+
+    const updates: Record<string, unknown> = {
+      health_status: result.status,
+      health_last_checked_at: nowIso,
+      health_last_latency_ms: result.latency_ms,
+      health_last_error: result.error_message,
+      health_next_check_at: nextCheckAtIso,
+      health_lock_until: null,
+      health_lock_token: null,
+    };
+
+    if (result.status === "ok") {
+      updates.health_last_ok_at = nowIso;
+    }
+
+    const { error: updateError } = await supabase
+      .from("plesk_instances")
+      .update(updates)
+      .eq("id", instance.id)
+      .eq("health_lock_token", lockToken);
+
+    if (updateError) {
+      throw new Error(updateError.message);
+    }
+
+    const action =
+      result.status === "ok"
+        ? "health_check_ok"
+        : result.status === "degraded"
+          ? "health_check_degraded"
+          : "health_check_failed";
+
+    await insertActionLogBestEffort(supabase, {
+      agency_id: instance.agency_id,
+      actor_user_id: actorUserId,
+      target_type: "plesk_instance",
+      target_id: instance.id,
+      action,
+      status: result.status === "down" ? "failed" : "success",
+      error_message: result.error_message,
+      metadata: {
+        instance_id: instance.id,
+        lock_token: lockToken,
+        source,
+        latency_ms: result.latency_ms,
+        duration_ms: Date.now() - startedAt,
+      },
+    });
+
+    return {
+      checked: true,
+      status: result.status,
+      latency_ms: result.latency_ms,
+      error_message: result.error_message,
+    };
+  } catch (error) {
+    const errorMessage =
+      error instanceof Error
+        ? error.message.slice(0, 1000)
+        : "health_check_failed";
+
+    await supabase
+      .from("plesk_instances")
+      .update({
+        health_status: "down",
+        health_last_checked_at: nowIso,
+        health_last_error: errorMessage,
+        health_next_check_at: nextCheckAtIso,
+      })
+      .eq("id", instance.id);
+
+    await insertActionLogBestEffort(supabase, {
+      agency_id: instance.agency_id,
+      actor_user_id: actorUserId,
+      target_type: "plesk_instance",
+      target_id: instance.id,
+      action: "health_check_failed",
+      status: "failed",
+      error_message: errorMessage,
+      metadata: {
+        instance_id: instance.id,
+        lock_token: lockToken,
+        source,
+        duration_ms: Date.now() - startedAt,
+      },
+    });
+
+    return {
+      checked: true,
+      status: "down",
+      latency_ms: Date.now() - startedAt,
+      error_message: errorMessage,
+    };
+  } finally {
+    await supabase.rpc("release_instance_health_lock", {
+      p_instance_id: instance.id,
+      p_lock_token: lockToken,
+    });
+  }
+}
+
+export async function runHealthCheckTick(
+  supabase: SupabaseLike,
+): Promise<HealthTickSummary> {
+  const startedAt = Date.now();
+  const now = Date.now();
+
+  const { data: rows, error } = await supabase
+    .from("plesk_instances")
+    .select(
+      "id, agency_id, base_url, auth_type, encrypted_secret, health_enabled, health_check_frequency_minutes, health_next_check_at, health_lock_until",
+    )
+    .eq("health_enabled", true)
+    .order("health_next_check_at", { ascending: true, nullsFirst: true })
+    .limit(50);
+
+  if (error) {
+    throw new Error(error.message);
+  }
+
+  const eligible = ((rows ?? []) as HealthInstance[])
+    .filter((instance) => isDue(instance, now) && isUnlocked(instance, now))
+    .slice(0, MAX_INSTANCES_PER_TICK);
+
+  let checked = 0;
+  let ok = 0;
+  let degraded = 0;
+  let failed = 0;
+
+  const queue = [...eligible];
+
+  const workers = Array.from(
+    { length: Math.min(MAX_CONCURRENCY, queue.length) },
+    async () => {
+      while (queue.length > 0) {
+        const instance = queue.shift();
+        if (!instance) break;
+
+        const result = await runInstanceHealthCheck(supabase, instance, {
+          actorUserId: null,
+          force: false,
+          source: "cron",
+        });
+
+        if (!result.checked || !result.status) continue;
+
+        checked += 1;
+        if (result.status === "ok") ok += 1;
+        else if (result.status === "degraded") degraded += 1;
+        else failed += 1;
+      }
+    },
+  );
+
+  await Promise.all(workers);
+
+  return {
+    checked,
+    ok,
+    degraded,
+    failed,
+    duration_ms: Date.now() - startedAt,
+  };
+}

lib/types.ts

@@ -89,6 +89,16 @@ export type Database = {
           next_auto_sync_at: string | null;
           auto_sync_lock_until: string | null;
           auto_sync_lock_token: string | null;
+          health_enabled: boolean;
+          health_status: "ok" | "degraded" | "down" | "unknown";
+          health_last_checked_at: string | null;
+          health_last_ok_at: string | null;
+          health_last_error: string | null;
+          health_last_latency_ms: number | null;
+          health_check_frequency_minutes: number;
+          health_next_check_at: string | null;
+          health_lock_until: string | null;
+          health_lock_token: string | null;
           created_at: string;
         };
         Insert: {
@@ -113,6 +123,16 @@ export type Database = {
           next_auto_sync_at?: string | null;
           auto_sync_lock_until?: string | null;
           auto_sync_lock_token?: string | null;
+          health_enabled?: boolean;
+          health_status?: "ok" | "degraded" | "down" | "unknown";
+          health_last_checked_at?: string | null;
+          health_last_ok_at?: string | null;
+          health_last_error?: string | null;
+          health_last_latency_ms?: number | null;
+          health_check_frequency_minutes?: number;
+          health_next_check_at?: string | null;
+          health_lock_until?: string | null;
+          health_lock_token?: string | null;
           created_at?: string;
         };
         Update: {
@@ -137,6 +157,16 @@ export type Database = {
           next_auto_sync_at?: string | null;
           auto_sync_lock_until?: string | null;
           auto_sync_lock_token?: string | null;
+          health_enabled?: boolean;
+          health_status?: "ok" | "degraded" | "down" | "unknown";
+          health_last_checked_at?: string | null;
+          health_last_ok_at?: string | null;
+          health_last_error?: string | null;
+          health_last_latency_ms?: number | null;
+          health_check_frequency_minutes?: number;
+          health_next_check_at?: string | null;
+          health_lock_until?: string | null;
+          health_lock_token?: string | null;
           created_at?: string;
         };
       };

supabase/migrations/CL38_instance_health_monitoring.sql

@@ -0,0 +1,78 @@
+-- CL8: instance health monitoring fields + lock helpers
+
+alter table public.plesk_instances
+  add column if not exists health_enabled boolean not null default true,
+  add column if not exists health_status text not null default 'unknown',
+  add column if not exists health_last_checked_at timestamptz,
+  add column if not exists health_last_ok_at timestamptz,
+  add column if not exists health_last_error text,
+  add column if not exists health_last_latency_ms integer,
+  add column if not exists health_check_frequency_minutes integer not null default 30,
+  add column if not exists health_next_check_at timestamptz,
+  add column if not exists health_lock_until timestamptz,
+  add column if not exists health_lock_token text;
+
+alter table public.plesk_instances
+  drop constraint if exists plesk_instances_health_status_check;
+
+alter table public.plesk_instances
+  add constraint plesk_instances_health_status_check
+  check (health_status in ('ok', 'degraded', 'down', 'unknown'));
+
+alter table public.plesk_instances
+  drop constraint if exists plesk_instances_health_check_frequency_minutes_check;
+
+alter table public.plesk_instances
+  add constraint plesk_instances_health_check_frequency_minutes_check
+  check (health_check_frequency_minutes between 5 and 10080);
+
+create index if not exists idx_plesk_instances_health_due
+  on public.plesk_instances (health_enabled, health_next_check_at);
+
+create or replace function public.acquire_instance_health_lock(
+  p_instance_id uuid,
+  p_lock_token text,
+  p_ttl_seconds integer default 300,
+  p_force boolean default false
+)
+returns boolean
+language plpgsql
+security definer
+set search_path = public
+as $$
+declare
+  v_acquired boolean := false;
+begin
+  update public.plesk_instances
+  set
+    health_lock_until = now() + make_interval(secs => p_ttl_seconds),
+    health_lock_token = p_lock_token
+  where id = p_instance_id
+    and health_enabled = true
+    and (
+      p_force = true
+      or health_next_check_at is null
+      or health_next_check_at <= now()
+    )
+    and (health_lock_until is null or health_lock_until < now());
+
+  get diagnostics v_acquired = row_count;
+  return v_acquired;
+end;
+$$;
+
+create or replace function public.release_instance_health_lock(
+  p_instance_id uuid,
+  p_lock_token text
+)
+returns void
+language sql
+security definer
+set search_path = public
+as $$
+  update public.plesk_instances
+  set health_lock_until = null,
+      health_lock_token = null
+  where id = p_instance_id
+    and health_lock_token = p_lock_token;
+$$;