openai-realtime

// Connect to OpenAI Realtime API
import WebSocket from "ws";

const ws = new WebSocket("wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview", {
  headers: {
    "Authorization": `Bearer ${process.env.OPENAI_API_KEY}`,
    "OpenAI-Beta": "realtime=v1",
  },
});

ws.on("open", () => {
  // Configure the session
  ws.send(JSON.stringify({
    type: "session.update",
    session: {
      modalities: ["text", "audio"],
      voice: "alloy",                      // alloy, echo, fable, onyx, nova, shimmer
      instructions: `You are a helpful dental clinic receptionist named Ava.
        Be warm, professional, and concise. Use short sentences appropriate for phone calls.
        If asked about medical advice, say you'll transfer to the dentist.`,
      input_audio_format: "pcm16",         // 16-bit PCM, 24kHz
      output_audio_format: "pcm16",
      input_audio_transcription: {
        model: "whisper-1",                 // Also transcribe for logging
      },
      turn_detection: {
        type: "server_vad",                 // Server-side voice activity detection
        threshold: 0.5,                     // Sensitivity (0-1)
        prefix_padding_ms: 300,             // Include 300ms before speech start
        silence_duration_ms: 500,           // 500ms silence = end of turn
      },
      tools: [                              // Function calling tools
        {
          type: "function",
          name: "check_availability",
          description: "Check available appointment slots",
          parameters: {
            type: "object",
            properties: {
              date: { type: "string", description: "Date in YYYY-MM-DD format" },
              procedure: { type: "string", enum: ["cleaning", "filling", "crown", "consultation"] },
            },
            required: ["date", "procedure"],
          },
        },
        {
          type: "function",
          name: "book_appointment",
          description: "Book an appointment for a patient",
          parameters: {
            type: "object",
            properties: {
              patient_name: { type: "string" },
              phone: { type: "string" },
              date: { type: "string" },
              time: { type: "string" },
              procedure: { type: "string" },
            },
            required: ["patient_name", "date", "time", "procedure"],
          },
        },
      ],
    },
  }));
});

// Handle events from OpenAI
ws.on("message", (data) => {
  const event = JSON.parse(data.toString());

  switch (event.type) {
    case "response.audio.delta":
      // Stream audio chunks to speaker/WebRTC
      const audioChunk = Buffer.from(event.delta, "base64");
      sendToSpeaker(audioChunk);
      break;

    case "response.audio_transcript.delta":
      // Real-time transcript of AI's response
      process.stdout.write(event.delta);
      break;

    case "conversation.item.input_audio_transcription.completed":
      // User's speech transcribed
      console.log(`\nUser said: ${event.transcript}`);
      break;

    case "response.function_call_arguments.done":
      // AI wants to call a function
      handleFunctionCall(event.name, JSON.parse(event.arguments));
      break;

    case "input_audio_buffer.speech_started":
      // User started speaking — interrupt AI if it's talking
      console.log("[User interruption detected]");
      break;
  }
});

// Send microphone audio
function sendAudio(pcmBuffer: Buffer) {
  ws.send(JSON.stringify({
    type: "input_audio_buffer.append",
    audio: pcmBuffer.toString("base64"),
  }));
}

// Handle function calls
async function handleFunctionCall(name: string, args: any) {
  let result: string;

  if (name === "check_availability") {
    const slots = await checkClinicSlots(args.date, args.procedure);
    result = JSON.stringify(slots);
  } else if (name === "book_appointment") {
    const booking = await createAppointment(args);
    result = JSON.stringify(booking);
  }

  // Send function result back — AI will speak the response
  ws.send(JSON.stringify({
    type: "conversation.item.create",
    item: {
      type: "function_call_output",
      call_id: event.call_id,
      output: result,
    },
  }));

  // Trigger AI to respond with the function result
  ws.send(JSON.stringify({ type: "response.create" }));
}

# Using OpenAI Python SDK
from openai import AsyncOpenAI

client = AsyncOpenAI()

async def run_voice_agent():
    async with client.beta.realtime.connect(
        model="gpt-4o-realtime-preview"
    ) as connection:
        await connection.session.update(session={
            "modalities": ["text", "audio"],
            "voice": "nova",
            "instructions": "You are a helpful assistant.",
            "turn_detection": {"type": "server_vad"},
        })

        # Send audio from microphone
        await connection.input_audio_buffer.append(audio=base64_audio)

        # Process events
        async for event in connection:
            if event.type == "response.audio.delta":
                play_audio(event.delta)
            elif event.type == "response.done":
                print("AI finished speaking")