Realtime Voice & Avatars

pnpm add glove-voice-s2s

import { RealtimeAgent, s2sDrivenModel } from "glove-voice-s2s";

// The cleanest form: the model slot carries the realtime config, and
// RealtimeAgent derives the provider session from the agent itself.
const agent = new Glove({
  model: s2sDrivenModel({
    label: "s2s-front",
    provider: "openai",                                        // or S2S_PROVIDER
    voice: "marin",                                            // or S2S_VOICE
    turnDetection: { type: "semantic_vad", eagerness: "low" }, // typed knobs
  }),
  systemPrompt, store, displayManager, compaction_config,
}).fold(myTool).build();

const rt = new RealtimeAgent({ agent });
await rt.start();

const rt = new RealtimeAgent({
  agent,                                    // a built Glove (IGloveRunnable)
  adapter: createS2SAdapter({ provider: "gemini" }),
  instructions: SPOKEN_PERSONA,             // re-voice the text prompt for speech
  excludeTools: ["render_chart"],           // withhold tools that don't belong in a call
});

rt.on("user_said", (t) => log("caller:", t));
rt.on("agent_said", (t) => log("agent:", t));
await rt.start();

// transport mode: wire audio yourself
micStream.on("pcm", (pcm) => rt.sendAudio(pcm));
rt.adapter.on("audio", (pcm, format) => speaker.play(pcm, format.sampleRate));

// push an async result into the live call — the model relays it out loud
rt.inject("the lookup finished: covered until 2031", { respond: true });

// OpenAI — the model judges WHETHER you were done
createS2SAdapter({ provider: "openai", turnDetection: {
  type: "semantic_vad",
  eagerness: "low",              // low | medium | high | auto
}});

// …or threshold-driven
createS2SAdapter({ provider: "openai", turnDetection: {
  type: "server_vad",
  threshold: 0.6,                // how loud counts as speech
  silence_duration_ms: 700,      // trailing silence before end-of-turn
  prefix_padding_ms: 300,
  idle_timeout_ms: 10_000,
}});
// turnDetection: null → manual / push-to-talk

// Gemini
createS2SAdapter({ provider: "gemini", realtimeInput: {
  automaticActivityDetection: {
    startOfSpeechSensitivity: "START_SENSITIVITY_LOW",
    endOfSpeechSensitivity: "END_SENSITIVITY_LOW",
    silenceDurationMs: 700,
  },
  activityHandling: "NO_INTERRUPTION",   // default is barge-in
}});

import { RealtimeAgent, s2sDrivenModel } from "glove-voice-s2s";

// Use this model in your existing Glove agent definition.
const model = s2sDrivenModel({
  provider: "openai-live",
  model: "gpt-live-1",             // default voice model
  backendModel: "gpt-5.6-luna",     // independent Responses model
  voice: "marin",
  sampleRate: 24000,               // or 16000, in both directions
  instructions: "Speak briefly. Delegate lookups before answering.",
});
// Build agent with this model, your tools, and your system prompt.
const rt = new RealtimeAgent({ agent });
rt.on("transcript", ({ role, delta, startMs, endMs }) => {
  // Preserve timed fragments and allow overlapping speakers.
});
rt.on("usage", ({ seconds, final }) => {
  // Cumulative duration: replace the previous snapshot, do not sum it.
});
await rt.start();
// Host supplies continuous paced PCM, including silence, and plays audio events.
// On teardown, await rt.stop() before releasing the host resources.

import { createOpenAIRealtimeToken } from "glove-voice-s2s/server";

const { token } = await createOpenAIRealtimeToken({
  apiKey: process.env.OPENAI_API_KEY!,
  instructions: SPOKEN_PERSONA,
  voice: "marin",
  tools: [{ name: "delegate_to_worker", description: "…", parameters: {} }],
});

import { OpenAIRealtimeAdapter } from "glove-voice-s2s";

const s2s = new OpenAIRealtimeAdapter({
  getToken: () => fetchToken("/api/voice/s2s-token"),
});

s2s.on("tool_call", async ({ callId, name, arguments: args }) => {
  const result = await runWorker(JSON.parse(args).request);  // your heavy agent
  s2s.sendToolResult(callId, result);                        // relayed out loud
});

await s2s.connect();

mic ──▶ S2S model (brain + voice) ──▶ agent PCM ──▶ AvatarAdapter ──▶ the face
          │ tool calls unchanged                          (provider WebRTC surface)
          ▼
     worker over the mesh

pnpm add glove-voice-avatar

import { RealtimeAgent } from "glove-voice-s2s";
import { TavusEchoAdapter, attachAvatar } from "glove-voice-avatar";

const rt = new RealtimeAgent({ agent });   // the voice stack, exactly as before
await rt.start();

const avatar = new TavusEchoAdapter({
  apiKey: process.env.TAVUS_API_KEY!,      // server-side only
  faceId: process.env.TAVUS_FACE_ID!,
  // palId omitted → ensureEchoPal() reuses-or-creates a MINIMAL echo PAL
  // (no greeting, no TTS layer) so the ONLY voice is ever the agent's.
  sendInteraction: (event) => duct.send({ t: "avatar_interaction", event }),
});

const detach = await attachAvatar(rt, avatar);

avatar.view; // { kind: "webrtc-room", url: "https://…" } — hand to the client

pnpm add glove-voice-livekit

import { LiveKitTransport, attachRealtime, mintParticipantToken } from "glove-voice-livekit";

const transport = new LiveKitTransport({
  url: process.env.LIVEKIT_URL!,
  token: await mintParticipantToken(
    { apiKey: process.env.LIVEKIT_API_KEY!, apiSecret: process.env.LIVEKIT_API_SECRET! },
    { roomName: "call-42", identity: "agent" },
  ),
});

await transport.connect();
attachRealtime(rt, transport);   // mics → model, model → track, interrupt → flush
await rt.start();

import { TavusLiveKitAvatar, mintAvatarToken } from "glove-voice-livekit";
import { attachAvatar } from "glove-voice-avatar";

// The avatar publishes the voice on the agent's behalf — don't double it.
const transport = new LiveKitTransport({ url, token, publishAgentAudio: false });
await transport.connect();
attachRealtime(rt, transport, { agentAudio: false });

const avatar = new TavusLiveKitAvatar({
  apiKey: process.env.TAVUS_API_KEY!,
  faceId: process.env.TAVUS_FACE_ID!,   // minimal echo PAL ensured automatically
  livekitUrl: url,
  avatarToken: await mintAvatarToken(creds, { roomName: "call-42" }),
});

await attachAvatar(rt, avatar);