Voice agents — pipeline, engine, and realtime

// lib/agent.ts — shared by the route and the client. Safe to import in a
// client component: samai-sdk/voice pulls in no Node built-ins.
import { defineTool, defineVoiceAgent } from "samai-sdk/voice";
import { z } from "zod";

export const getTimeTool = defineTool({
  name: "get_time",
  description: "Gets the current time in the user's timezone",
  parameters: z.object({ timezone: z.string().optional() }),
  execute: async ({ timezone }) => new Date().toLocaleTimeString("en-US", { timeZone: timezone }),
});

export const agent = defineVoiceAgent({
  name: "assistant",
  instructions: "You are a helpful, concise voice assistant.",
  model: "gpt-realtime",
  tools: [getTimeTool],
});

// app/api/realtime-secret/route.ts — SERVER ONLY.
// This is the one place the long-lived API key is allowed to appear.
import { NextResponse } from "next/server";
import { createRealtimeClientSecret } from "samai-sdk/voice";

export async function POST() {
  const { value, expiresAt } = await createRealtimeClientSecret({
    apiKey: process.env.OPENAI_API_KEY!,
    model: "gpt-realtime",
    ttlSeconds: 600,             // 10–7200; the browser holds it for this long
    safetyIdentifier: "user-1234", // OpenAI's abuse-tracking header
  });
  return NextResponse.json({ clientSecret: value, expiresAt });
}

"use client";
// components/VoiceAgent.tsx
import { useState } from "react";
import { openaiRealtime, stopMediaStream, type VoiceAgentEvent } from "samai-sdk/voice";

export function VoiceAgent() {
  const [status, setStatus] = useState("idle");
  const [caption, setCaption] = useState("");
  const audioRef = useRef<HTMLAudioElement>(null);
  const sessionRef = useRef<any>(null);
  const streamRef = useRef<MediaStream | null>(null);

  async function start() {
    // 1. Mint the ephemeral secret on the server. Never ship OPENAI_API_KEY here.
    const { clientSecret } = await fetch("/api/realtime-secret", { method: "POST" })
      .then((r) => r.json());

    // 2. Ask for the mic ourselves, so permission errors stay ours to surface.
    const inputStream = await navigator.mediaDevices.getUserMedia({ audio: true });
    streamRef.current = inputStream;

    // 3. Connect over WebRTC. transport: "webrtc" is implied by inputStream.
    const provider = openaiRealtime({ clientSecret, inputStream });
    const session = await provider.connect({ agent, session });
    sessionRef.current = session;

    // 4. The assistant's voice arrives as a MediaStream — no PCM plumbing needed.
    if (audioRef.current && session.getRemoteStream()) {
      audioRef.current.srcObject = session.getRemoteStream();
      await audioRef.current.play();
    }

    session.on("assistant-transcript-delta", (e: any) => setCaption((c) => c + e.delta));
    session.on("user-transcript-delta", (e: any) => setCaption((c) => c + e.delta));
    session.on("connection-state", (e: any) => setStatus(e.state));
  }

  async function stop() {
    await sessionRef.current?.close();
    stopMediaStream(streamRef.current); // release the microphone
    sessionRef.current = null;
    streamRef.current = null;
    setStatus("idle");
  }

  return (
    <div>
      <button onClick={status === "idle" ? start : stop}>{status === "idle" ? "Talk" : status}</button>
      <p>{caption}</p>
      <audio ref={audioRef} autoPlay />
    </div>
  );
}

import { pipelineVoice } from "samai-sdk/voice";
import { deepgramSTT } from "samai-sdk/voice";
import { elevenLabsTTS } from "samai-sdk/voice";
import { defineVoiceAgent } from "samai-sdk/voice";
import { anthropic } from "samai-sdk";

// Any text Provider works as the LLM — swap anthropic() for openai() etc.
const voiceProvider = pipelineVoice({
  stt: deepgramSTT({ apiKey: process.env.DEEPGRAM_KEY }),
  llm: anthropic({ apiKey: process.env.ANTHROPIC_KEY }),
  tts: elevenLabsTTS({ apiKey: process.env.ELEVEN_KEY }),
});

const agent = defineVoiceAgent({
  name: "concierge",
  instructions: "You are a helpful, concise voice assistant.",
  model: "claude-sonnet-4-5",
  voice: { interruption: "confidence-gated", backchannel: true },
});

const session = await voiceProvider.connect({ agent, session });
session.on("user-speech-ended", (e) => console.log("user:", e.transcript));
session.on("interruption", () => console.log("barge-in — TTS cancelled"));
session.sendAudio(micChunk); // ArrayBuffer from your mic
session.interrupt();         // explicit barge-in
await session.close();

import { ConversationEngine } from "samai-sdk/voice";

const engine = new ConversationEngine({ agent, session, trace });
// engine.handleUserSpeechEnded(transcript, confidence) -> clarification or null
// engine.handleBargeIn({ confidence, durationMs })     -> true if barge-in accepted
// engine.getState() // "idle" | "connecting" | "listening" | "user_speaking" | "thinking"
//                   // | "speaking" | "assistant_speaking" | "interrupting" | "interrupted"
//                   // | "reconnecting" | "error"

import { useVoiceAgent } from "samai-sdk/react-voice";
import { pipelineVoice } from "samai-sdk/voice";

function VoiceButton({ provider, agent }) {
  const { isConnected, isSpeaking, transcript, connect, disconnect, sendAudio, interrupt } =
    useVoiceAgent(provider, agent);
  return (
    <button onClick={isConnected ? disconnect : connect}>
      {isSpeaking ? "Speaking…" : isConnected ? "Listening" : "Connect"}
    </button>
  );
}

import { generateSpeech, transcribeAudio } from "samai-sdk";
import { writeFile, readFile } from "node:fs/promises";

const { audio } = await generateSpeech({ input: "Hello there!", voice: "nova" });
await writeFile("out.mp3", audio);

const { text } = await transcribeAudio({ audio: await readFile("recording.mp3"), filename: "recording.mp3" });

import { createRealtimeSession } from "samai-sdk";

// Server-side, or any host with a WebSocket-capable runtime.
const session = createRealtimeSession({
  instructions: "You are a helpful, concise voice assistant.",
  voice: "alloy",
  tools: [getWeatherTool],
});

session.on((event) => {
  if (event.type === "audio.delta") playAudioChunk(event.audio);
  if (event.type === "speech_started") session.truncateLastResponse(playedMs);
});

await session.connect();
session.sendText("What's the weather in Tokyo?");
session.interrupt();
await session.close();