Build Browser Automation for AI Agents

npm install puppeteer zod ioredis
npm install -D typescript @types/node

// src/browser/automation.ts — browser sessions for AI agents (Puppeteer + Redis metadata)
import { lookup } from "node:dns/promises";
import { randomBytes } from "node:crypto";
import { isIP } from "node:net";
import { Redis } from "ioredis";
import puppeteer, { type Browser, type Page } from "puppeteer";
import { z } from "zod";

const redis = new Redis(process.env.REDIS_URL!);
const MAX_SESSIONS = 5;
const IDLE_MS = 30 * 60 * 1000;

// Agents send JSON; validate it before it reaches the browser
export const ActionSchema = z.discriminatedUnion("type", [
  z.object({ type: z.literal("navigate"), url: z.string().url() }),
  z.object({ type: z.literal("click"), selector: z.string().min(1) }),
  z.object({ type: z.literal("type"), selector: z.string().min(1), value: z.string() }),
  z.object({ type: z.literal("select"), selector: z.string().min(1), value: z.string() }),
  z.object({ type: z.literal("screenshot") }),
  z.object({ type: z.literal("extract"), schema: z.record(z.string(), z.string()).optional() }),
  z.object({ type: z.literal("scroll"), pixels: z.number().int().default(500) }),
  z.object({ type: z.literal("wait"), ms: z.number().int().min(0).max(10_000).default(1000) }),
]);
export type BrowserAction = z.infer<typeof ActionSchema>;

export interface ActionResult {
  success: boolean;
  action: string;
  data?: unknown;
  screenshot?: string; // base64 PNG
  error?: string;
  pageState: { url: string; title: string };
}

// The agent controls the URL, so refuse anything that is not public http(s):
// otherwise it can reach localhost, the cloud metadata service or your intranet.
function isPrivateIp(ip: string): boolean {
  if (isIP(ip) === 6) {
    const v = ip.toLowerCase();
    // URL parsing turns ::ffff:127.0.0.1 into ::ffff:7f00:1, so decode the hex form too
    const mapped = v.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
    if (mapped) {
      const hi = parseInt(mapped[1], 16), lo = parseInt(mapped[2], 16);
      return isPrivateIp(`${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`);
    }
    return v === "::1" || v === "::" || v.startsWith("fc") || v.startsWith("fd") || /^fe[89ab]/.test(v) ||
      (v.startsWith("::ffff:") && isPrivateIp(v.slice(7)));
  }
  const [a, b] = ip.split(".").map(Number);
  return a === 0 || a === 10 || a === 127 || (a === 169 && b === 254) ||
    (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127);
}

export async function assertPublicUrl(raw: string): Promise<void> {
  if (process.env.BROWSER_ALLOW_PRIVATE === "1") return; // local testing only
  const url = new URL(raw);
  if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`Blocked protocol ${url.protocol}`);
  const host = url.hostname.replace(/^\[|\]$/g, "");
  const addrs = isIP(host) ? [{ address: host }] : await lookup(host, { all: true });
  if (addrs.some((a) => isPrivateIp(a.address))) throw new Error(`Blocked non-public address for ${host}`);
}

let browser: Browser | null = null;
const sessions = new Map<string, { page: Page; idle: NodeJS.Timeout }>();

async function ensureBrowser(): Promise<Browser> {
  if (!browser || !browser.connected) {
    browser = await puppeteer.launch({
      headless: true,
      // Keep Chrome's sandbox on. Only containers that run as root need it off.
      args: process.env.BROWSER_NO_SANDBOX === "1" ? ["--no-sandbox"] : [],
    });
  }
  return browser;
}

function touch(id: string) {
  const s = sessions.get(id);
  if (!s) return;
  clearTimeout(s.idle);
  s.idle = setTimeout(() => void closeSession(id), IDLE_MS);
}

export async function createSession(): Promise<{ id: string }> {
  if (sessions.size >= MAX_SESSIONS) throw new Error("Too many open sessions");
  const page = await (await ensureBrowser()).newPage();
  await page.setViewport({ width: 1280, height: 800 });

  // Check every navigation request, redirects included, and skip heavy resources
  await page.setRequestInterception(true);
  page.on("request", (req) => {
    if (["image", "font", "media"].includes(req.resourceType())) return void req.abort();
    if (req.isNavigationRequest()) {
      assertPublicUrl(req.url()).then(() => req.continue(), () => req.abort("blockedbyclient"));
    } else {
      req.continue();
    }
  });

  const id = `bs-${randomBytes(6).toString("hex")}`;
  sessions.set(id, { page, idle: setTimeout(() => void closeSession(id), IDLE_MS) });
  await redis.set(`browser:session:${id}`, JSON.stringify({ createdAt: new Date().toISOString() }), "EX", 3600);
  return { id };
}

export async function executeActions(sessionId: string, input: unknown[]): Promise<ActionResult[]> {
  const entry = sessions.get(sessionId);
  if (!entry) throw new Error("Session not found");
  const { page } = entry;
  const state = async () => ({ url: page.url(), title: await page.title() });
  const results: ActionResult[] = [];

  for (const raw of input) {
    touch(sessionId);
    const parsed = ActionSchema.safeParse(raw);
    if (!parsed.success) {
      results.push({ success: false, action: "invalid", error: parsed.error.message, pageState: await state() });
      continue;
    }
    const a = parsed.data;
    try {
      let data: unknown;
      let screenshot: string | undefined;
      switch (a.type) {
        case "navigate":
          await assertPublicUrl(a.url);
          await page.goto(a.url, { waitUntil: "networkidle2", timeout: 15_000 });
          break;
        case "click":
          await page.locator(a.selector).setTimeout(5000).click();
          await page.waitForNetworkIdle({ timeout: 3000 }).catch(() => {});
          break;
        case "type":
          // fill() replaces the field's current value, so no triple-click needed
          await page.locator(a.selector).setTimeout(5000).fill(a.value);
          break;
        case "select":
          await page.select(a.selector, a.value);
          break;
        case "screenshot":
          screenshot = await page.screenshot({ type: "png", encoding: "base64" });
          break;
        case "extract":
          if (a.schema) {
            data = Object.fromEntries(await Promise.all(Object.entries(a.schema).map(async ([key, sel]) =>
              [key, await page.$$eval(sel, (els) => els.map((el) => el.textContent?.trim()))] as const)));
          } else {
            data = { text: await page.evaluate(() => document.body.innerText) };
          }
          break;
        case "scroll":
          await page.evaluate((px) => window.scrollBy(0, px), a.pixels);
          break;
        case "wait":
          await new Promise((r) => setTimeout(r, a.ms));
          break;
      }
      results.push({ success: true, action: a.type, data, screenshot, pageState: await state() });
    } catch (error) {
      results.push({ success: false, action: a.type, error: (error as Error).message, pageState: await state() });
    }
  }
  return results;
}

// Snapshot of what an agent can act on. Each element gets a data-agent-id so the
// returned selector can be passed straight back to click/type/select.
export async function getPageState(sessionId: string) {
  const entry = sessions.get(sessionId);
  if (!entry) throw new Error("Session not found");
  const { page } = entry;
  touch(sessionId);

  const interactiveElements = await page.evaluate(() => {
    const els = document.querySelectorAll("a, button, input, select, textarea, [role='button']");
    return Array.from(els).slice(0, 100).map((el, i) => {
      el.setAttribute("data-agent-id", String(i));
      const rect = el.getBoundingClientRect();
      return {
        selector: `[data-agent-id="${i}"]`,
        type: el.tagName.toLowerCase(),
        text: (el.textContent || el.getAttribute("placeholder") || el.getAttribute("aria-label") || "").trim().slice(0, 100),
        visible: rect.width > 0 && rect.height > 0,
      };
    });
  });
  return { url: page.url(), title: await page.title(), interactiveElements };
}

export async function closeSession(sessionId: string): Promise<void> {
  const entry = sessions.get(sessionId);
  if (entry) {
    clearTimeout(entry.idle);
    sessions.delete(sessionId);
    await entry.page.close().catch(() => {});
  }
  await redis.del(`browser:session:${sessionId}`);
}

export async function shutdown(): Promise<void> {
  await Promise.all([...sessions.keys()].map(closeSession));
  await browser?.close();
  redis.disconnect();
}

import { createSession, executeActions, getPageState, closeSession } from "./automation";

const { id } = await createSession();
try {
  await executeActions(id, [{ type: "navigate", url: "https://example.com/pricing" }]);
  const page = await getPageState(id);          // give this to the model
  const [result] = await executeActions(id, [   // the model's chosen actions
    { type: "extract", schema: { plans: "h2", prices: ".price" } },
  ]);
  console.log(page.title, result.data);
} finally {
  await closeSession(id);
}