Build an AI Agent Evaluation Framework

npm install hono @hono/node-server zod pg ioredis
npm install -D typescript @types/node @types/pg

CREATE TABLE eval_cases (
  id          text PRIMARY KEY,
  suite       text NOT NULL,
  name        text NOT NULL,
  input       jsonb NOT NULL,   -- {"messages":[{"role":"user","content":"..."}]}
  expected    jsonb NOT NULL,   -- {"mustContain":[],"mustNotContain":[],"expectedTools":[],"maxLatencyMs":8000}
  weight      int NOT NULL DEFAULT 1
);

CREATE TABLE eval_runs (
  id            text PRIMARY KEY,
  suite         text NOT NULL,
  agent_version text NOT NULL,
  summary       jsonb NOT NULL,
  results       jsonb NOT NULL,
  completed_at  timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX eval_runs_suite_time ON eval_runs (suite, completed_at DESC);

// src/eval.ts
import { randomBytes } from "node:crypto";
import { Pool } from "pg";
import { Redis } from "ioredis";

export const pool = new Pool({ connectionString: process.env.DATABASE_URL });
const redis = new Redis(process.env.REDIS_URL!);

export type Message = { role: "user" | "assistant"; content: string };
export type AgentFn = (messages: Message[]) => Promise<{ response: string; toolCalls: string[] }>;

interface Expected {
  mustContain?: string[];
  mustNotContain?: string[];
  expectedTools?: string[];
  maxLatencyMs?: number;
}
interface CaseRow { id: string; name: string; input: { messages: Message[] }; expected: Expected }
interface Result {
  caseId: string; passed: boolean; score: number; latencyMs: number;
  forbiddenClaim: boolean; toolsOk: boolean; issues: string[];
}
export interface Summary {
  total: number; passed: number; avgScore: number;
  forbiddenClaimRate: number; toolAccuracy: number; p95LatencyMs: number;
}

const has = (text: string, phrase: string) => text.toLowerCase().includes(phrase.toLowerCase());

function p95(values: number[]) {
  const sorted = [...values].sort((a, b) => a - b);
  return sorted.length ? sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)] : 0;
}

async function runCase(tc: CaseRow, agent: AgentFn): Promise<Result> {
  const issues: string[] = [];
  const start = Date.now();
  let score = 1;
  let forbiddenClaim = false;
  let toolsOk = true;
  try {
    const { response, toolCalls } = await agent(tc.input.messages);
    const latencyMs = Date.now() - start; // measured here, not trusted from the agent
    for (const p of tc.expected.mustContain ?? []) {
      if (!has(response, p)) { issues.push(`Missing: "${p}"`); score -= 0.2; }
    }
    for (const p of tc.expected.mustNotContain ?? []) {
      if (has(response, p)) { issues.push(`Forbidden claim: "${p}"`); forbiddenClaim = true; score -= 0.3; }
    }
    for (const t of tc.expected.expectedTools ?? []) {
      if (!toolCalls.includes(t)) { issues.push(`Tool not called: ${t}`); toolsOk = false; score -= 0.2; }
    }
    if (tc.expected.maxLatencyMs && latencyMs > tc.expected.maxLatencyMs) {
      issues.push(`Latency ${latencyMs}ms > ${tc.expected.maxLatencyMs}ms`); score -= 0.1;
    }
    score = Math.max(0, score);
    return { caseId: tc.id, passed: score >= 0.7 && !forbiddenClaim, score, latencyMs, forbiddenClaim, toolsOk, issues };
  } catch (err) {
    return {
      caseId: tc.id, passed: false, score: 0, latencyMs: Date.now() - start,
      forbiddenClaim: false, toolsOk: false, issues: [`Error: ${(err as Error).message}`],
    };
  }
}

export async function runEvalSuite(suite: string, agent: AgentFn, agentVersion: string) {
  const { rows } = await pool.query<CaseRow>(
    "SELECT id, name, input, expected FROM eval_cases WHERE suite = $1 ORDER BY weight DESC, id", [suite]);
  if (rows.length === 0) throw new Error(`No eval cases for suite "${suite}"`);

  const results: Result[] = [];
  for (const tc of rows) results.push(await runCase(tc, agent)); // sequential: keeps latency numbers honest

  const withTools = rows.filter((r) => r.expected.expectedTools?.length);
  const toolHits = results.filter((r, i) => rows[i].expected.expectedTools?.length && r.toolsOk).length;
  const summary: Summary = {
    total: results.length,
    passed: results.filter((r) => r.passed).length,
    avgScore: results.reduce((s, r) => s + r.score, 0) / results.length,
    forbiddenClaimRate: (results.filter((r) => r.forbiddenClaim).length / results.length) * 100,
    toolAccuracy: withTools.length ? (toolHits / withTools.length) * 100 : 100,
    p95LatencyMs: p95(results.map((r) => r.latencyMs)),
  };

  // Read the previous run before inserting this one
  const { rows: [prev] } = await pool.query<{ summary: Summary }>(
    "SELECT summary FROM eval_runs WHERE suite = $1 ORDER BY completed_at DESC LIMIT 1", [suite]);

  const id = `eval-${randomBytes(6).toString("hex")}`;
  await pool.query(
    "INSERT INTO eval_runs (id, suite, agent_version, summary, results) VALUES ($1, $2, $3, $4, $5)",
    [id, suite, agentVersion, JSON.stringify(summary), JSON.stringify(results)]);

  const regressed = !!prev && (
    summary.avgScore < prev.summary.avgScore - 0.05 ||
    summary.forbiddenClaimRate > prev.summary.forbiddenClaimRate + 2 ||
    summary.toolAccuracy < prev.summary.toolAccuracy - 5);
  if (regressed) {
    await redis.rpush("notification:queue", JSON.stringify({ type: "eval_regression", suite, agentVersion, current: summary, previous: prev.summary }));
  }
  return { id, summary, results, regressed };
}

export async function versionHistory(suite: string, limit = 10) {
  const { rows } = await pool.query(
    "SELECT agent_version AS version, summary, completed_at AS date FROM eval_runs WHERE suite = $1 ORDER BY completed_at DESC LIMIT $2",
    [suite, limit]);
  return rows;
}

// src/server.ts
import { Hono } from "hono";
import { serve } from "@hono/node-server";
import { z } from "zod";
import { runEvalSuite, versionHistory, type AgentFn } from "./eval";

const agentUnderTest: AgentFn = async (messages) => {
  // call your agent here and return its reply plus the names of the tools it invoked
  return { response: messages.at(-1)?.content ?? "", toolCalls: [] };
};

const app = new Hono();

app.post("/eval/run", async (c) => {
  const { suite, agentVersion } = z.object({ suite: z.string(), agentVersion: z.string() }).parse(await c.req.json());
  const run = await runEvalSuite(suite, agentUnderTest, agentVersion);
  return c.json({ id: run.id, regressed: run.regressed, summary: run.summary }, run.regressed ? 409 : 200);
});

app.get("/eval/:suite/history", async (c) => c.json(await versionHistory(c.req.param("suite"))));

serve({ fetch: app.fetch, hostname: "127.0.0.1", port: 3300 });