Build a Movie Companion

pnpm add glove-core glove-react glove-next glove-voice zod

pnpm add @ricky0123/vad-web onnxruntime-web

OPENROUTER_API_KEY=your-openrouter-key
TMDB_API_KEY=your-tmdb-v3-bearer-token
ELEVENLABS_API_KEY=your-elevenlabs-key

import { NextResponse } from "next/server";

const TMDB_API_BASE = "https://api.themoviedb.org/3";

export async function GET(
  req: Request,
  { params }: { params: Promise<{ path: string[] }> },
) {
  const { path } = await params;
  const url = new URL(req.url);

  const apiKey = process.env.TMDB_API_KEY;
  if (!apiKey) {
    return NextResponse.json({ error: "TMDB_API_KEY not set" }, { status: 500 });
  }

  const tmdbUrl = `${TMDB_API_BASE}/${path.join("/")}?${url.searchParams.toString()}`;

  const res = await fetch(tmdbUrl, {
    headers: { Authorization: `Bearer ${apiKey}` },
  });

  const data = await res.json();
  return NextResponse.json(data, { status: res.status });
}

const API_BASE = "/api/tmdb";
const TMDB_IMAGE_BASE = "https://image.tmdb.org/t/p";

export interface TMDBMovie {
  id: number;
  title: string;
  overview: string;
  release_date: string;
  vote_average: number;
  vote_count: number;
  poster_path: string | null;
  backdrop_path: string | null;
  genres?: { id: number; name: string }[];
  runtime?: number;
  tagline?: string;
  credits?: {
    cast: TMDBCastMember[];
    crew: TMDBCrewMember[];
  };
  videos?: { results: TMDBVideo[] };
  "watch/providers"?: {
    results: Record<string, TMDBProviderData>;
  };
}

// Image URL helpers
export function posterUrl(
  path: string | null,
  size: "w92" | "w154" | "w185" | "w342" | "w500" | "w780" | "original" = "w342",
): string | null {
  if (!path) return null;
  return `${TMDB_IMAGE_BASE}/${size}${path}`;
}

// Internal fetch helper — all calls go through the proxy
async function tmdbFetch<T>(path: string, params?: Record<string, string>): Promise<T> {
  const url = new URL(`${API_BASE}/${path}`, window.location.origin);
  if (params) {
    for (const [key, value] of Object.entries(params)) {
      url.searchParams.set(key, value);
    }
  }
  const res = await fetch(url.toString());
  if (!res.ok) {
    const errorBody = await res.text().catch(() => "Unknown error");
    throw new Error(`TMDB API error (${res.status}): ${errorBody}`);
  }
  return res.json() as Promise<T>;
}

// API functions
export async function searchMovies(query: string, year?: number): Promise<TMDBMovie[]> {
  const params: Record<string, string> = { query };
  if (year) params.year = String(year);
  const data = await tmdbFetch<{ results: TMDBMovie[] }>("search/movie", params);
  return data.results;
}

export async function getMovieDetails(movieId: number): Promise<TMDBMovie> {
  return tmdbFetch<TMDBMovie>(`movie/${movieId}`, {
    append_to_response: "credits,videos,watch/providers",
  });
}

// Utility helpers
export function movieYear(movie: TMDBMovie): string {
  if (!movie.release_date) return "Unknown";
  return movie.release_date.substring(0, 4);
}

export function getDirector(movie: TMDBMovie): string {
  if (!movie.credits?.crew) return "Unknown";
  const director = movie.credits.crew.find((c) => c.job === "Director");
  return director?.name ?? "Unknown";
}

export function getTopCast(movie: TMDBMovie, count: number = 5): TMDBCastMember[] {
  if (!movie.credits?.cast) return [];
  return movie.credits.cast.sort((a, b) => a.order - b.order).slice(0, count);
}

import { defineTool } from "glove-react";
import { z } from "zod";
import { searchMovies, posterUrl, movieYear, type TMDBMovie } from "../tmdb";

export function createSearchMoviesTool() {
  return defineTool({
    name: "search_movies",
    description:
      "Search for movies by title. Returns a visual grid of poster cards " +
      "and text results for narration.",
    inputSchema: z.object({
      query: z.string().describe("Search query for movies"),
      year: z.number().optional().describe("Filter by release year"),
      max_results: z.number().optional().default(4).describe("Max results (1-6)"),
    }),
    displayPropsSchema: z.object({
      movies: z.array(z.any()),
    }),

    async do(input, display) {
      const clampedMax = Math.max(1, Math.min(6, input.max_results ?? 4));
      const results = await searchMovies(input.query, input.year);
      const movies = results.slice(0, clampedMax);

      if (movies.length === 0) {
        return {
          status: "success" as const,
          data: `No movies found matching "${input.query}".`,
          renderData: { movies: [] },
        };
      }

      // Fire the visual card — does NOT block the LLM
      await display.pushAndForget({ movies });

      // Return descriptive text for voice narration
      const summaryLines = movies.map(
        (m, i) =>
          `${i + 1}. ${m.title} (${movieYear(m)}) — Rating: ${m.vote_average.toFixed(1)}/10`,
      );

      return {
        status: "success" as const,
        data: `Found ${movies.length} result(s) for "${input.query}":\n${summaryLines.join("\n")}`,
        renderData: { movies },
      };
    },

    render({ props }) {
      const movies = props.movies as TMDBMovie[];
      return (
        <div style={{ display: "flex", gap: 12, flexWrap: "wrap", justifyContent: "center" }}>
          {movies.map((movie) => (
            <PosterCard key={movie.id} movie={movie} />
          ))}
        </div>
      );
    },

    renderResult({ data }) {
      const result = data as { movies: TMDBMovie[] };
      return (
        <div style={{ display: "flex", gap: 12, flexWrap: "wrap", justifyContent: "center" }}>
          {result.movies.map((movie) => (
            <PosterCard key={movie.id} movie={movie} />
          ))}
        </div>
      );
    },
  });
}

import { defineTool } from "glove-react";
import { z } from "zod";
import {
  getMovieDetails,
  backdropUrl,
  movieYear,
  formatRuntime,
  getDirector,
  getTopCast,
  genreNames,
  type TMDBMovie,
} from "../tmdb";

export function createGetMovieDetailsTool() {
  return defineTool({
    name: "get_movie_details",
    description:
      "Get comprehensive details about a movie including overview, " +
      "cast, director, runtime, rating, genres, and streaming availability.",
    inputSchema: z.object({
      movie_id: z.number().describe("TMDB movie ID"),
    }),
    displayPropsSchema: z.object({
      movie: z.any(),
    }),

    async do(input, display) {
      const movie = await getMovieDetails(input.movie_id);
      await display.pushAndForget({ movie });

      const year = movieYear(movie);
      const director = getDirector(movie);
      const cast = getTopCast(movie, 5);
      const castNames = cast.map((c) => c.name).join(", ");
      const overviewSnippet =
        movie.overview.length > 200
          ? movie.overview.substring(0, 200) + "..."
          : movie.overview;

      return {
        status: "success" as const,
        data: `${movie.title} (${year}), directed by ${director}. ${overviewSnippet} Stars: ${castNames}. Rating: ${movie.vote_average.toFixed(1)}/10.`,
        renderData: { movie },
      };
    },

    render({ props }) {
      const movie = props.movie as TMDBMovie;
      return <MovieInfoCard movie={movie} />;
    },

    renderResult({ data }) {
      const result = data as { movie: TMDBMovie };
      return <MovieInfoCard movie={result.movie} />;
    },
  });
}

import { defineTool } from "glove-react";
import { z } from "zod";

export function createRememberPreferenceTool() {
  return defineTool({
    name: "remember_preference",
    description:
      "Remember a user preference about movies — favorite genres, " +
      "directors, actors, moods, or anything else. Data-only tool " +
      "with no visual display.",
    inputSchema: z.object({
      preference: z.string().describe("User preference to remember"),
      category: z
        .string()
        .optional()
        .describe("Category: genre, director, actor, mood, other"),
    }),
    displayPropsSchema: z.object({}),
    async do(input) {
      const category = input.category ?? "other";
      return {
        status: "success" as const,
        data: `Noted preference (${category}): ${input.preference}`,
      };
    },
  });
}

import type { ToolConfig } from "glove-react";
import { createSearchMoviesTool } from "./search-movies";
import { createGetMovieDetailsTool } from "./get-movie-details";
import { createGetRatingsTool } from "./get-ratings";
import { createGetTrailerTool } from "./get-trailer";
import { createCompareMoviesTool } from "./compare-movies";
import { createGetRecommendationsTool } from "./get-recommendations";
import { createGetPersonTool } from "./get-person";
import { createGetStreamingTool } from "./get-streaming";
import { createRememberPreferenceTool } from "./remember-preference";

export function createLolaTools(): ToolConfig[] {
  return [
    createSearchMoviesTool(),
    createGetMovieDetailsTool(),
    createGetRatingsTool(),
    createGetTrailerTool(),
    createCompareMoviesTool(),
    createGetRecommendationsTool(),
    createGetPersonTool(),
    createGetStreamingTool(),
    createRememberPreferenceTool(),
  ];
}

import { useMemo, type ReactNode } from "react";
import type { TimelineEntry, EnhancedSlot } from "glove-react";

interface VisualAreaProps {
  slots: EnhancedSlot[];
  timeline: TimelineEntry[];
  renderSlot: (slot: EnhancedSlot) => ReactNode;
  renderToolResult: (entry: TimelineEntry & { kind: "tool" }) => ReactNode;
  busy: boolean;
  onSuggestion?: (text: string) => void;
}

const SUGGESTIONS = [
  "Best sci-fi from the 90s",
  "Something like Eternal Sunshine",
  "Who directed Parasite?",
  "Cozy rainy day movies",
];

export function VisualArea({
  slots,
  timeline,
  renderSlot,
  renderToolResult,
  busy,
  onSuggestion,
}: VisualAreaProps) {
  const lastToolResult = useMemo(() => {
    for (let i = timeline.length - 1; i >= 0; i--) {
      const entry = timeline[i];
      if (
        entry.kind === "tool" &&
        entry.status === "success" &&
        entry.renderData !== undefined
      ) {
        return entry;
      }
    }
    return null;
  }, [timeline]);

  // Case 1: Active slots — render each via renderSlot
  if (slots.length > 0) {
    return (
      <div className="visual-area">
        {slots.map((slot) => (
          <div key={slot.id} className="display-card">
            {renderSlot(slot)}
          </div>
        ))}
      </div>
    );
  }

  // Case 2: Recent tool result with renderData
  if (lastToolResult) {
    const rendered = renderToolResult(lastToolResult);
    if (rendered) {
      return (
        <div className="visual-area">
          <div className="display-card">{rendered}</div>
        </div>
      );
    }
  }

  // Case 3: Empty state — suggestion chips
  if (!busy) {
    return (
      <div className="visual-area">
        <div className="lola-empty">
          <h1 className="lola-empty__title">Lola</h1>
          <p className="lola-empty__subtitle">
            Your voice-first movie companion.<br />
            Ask me anything about film.
          </p>
          {onSuggestion && (
            <div className="lola-empty__suggestions">
              {SUGGESTIONS.map((s) => (
                <button
                  key={s}
                  type="button"
                  className="lola-empty__chip"
                  onClick={() => onSuggestion(s)}
                >
                  {s}
                </button>
              ))}
            </div>
          )}
        </div>
      </div>
    );
  }

  return <div className="visual-area" />;
}

import { createVoiceTokenHandler } from "glove-next";

export const GET = createVoiceTokenHandler({ provider: "elevenlabs", type: "stt" });

import { createVoiceTokenHandler } from "glove-next";

export const GET = createVoiceTokenHandler({ provider: "elevenlabs", type: "tts" });

import { createElevenLabsAdapters } from "glove-voice";

async function fetchToken(path: string): Promise<string> {
  const res = await fetch(path);
  const data = (await res.json()) as { token?: string; error?: string };
  if (!res.ok || !data.token) {
    throw new Error(data.error ?? `Token fetch failed (${res.status})`);
  }
  return data.token;
}

export const { stt, createTTS } = createElevenLabsAdapters({
  getSTTToken: () => fetchToken("/api/voice/stt-token"),
  getTTSToken: () => fetchToken("/api/voice/tts-token"),
  voiceId: "XB0fDUnXU5powFXDhCwa", // "Charlotte" — warm, cinematic
});

export async function createSileroVAD() {
  const { SileroVADAdapter } = await import("glove-voice/silero-vad");
  const vad = new SileroVADAdapter({
    positiveSpeechThreshold: 0.5,
    negativeSpeechThreshold: 0.35,
    wasm: { type: "cdn" },
  });
  await vad.init();
  return vad;
}

const handleClick = () => {
  if (mode === "speaking") {
    onInterrupt();         // Tap while speaking → interrupt
  } else if (isProcessing) {
    onStop();              // Tap while processing → cancel
  } else if (isManual && mode === "listening") {
    if (isManualRecording) {
      onManualRecordStop();  // Tap while recording → send
    } else {
      onManualRecordStart(); // Tap while idle → start recording
    }
  } else {
    onStop();              // Tap otherwise → end voice session
  }
};

export const systemPrompt = `You are Lola, a passionate and knowledgeable movie companion.

## Your Personality
- Genuinely passionate about cinema across all genres and eras
- Warm but opinionated — you have taste but respect others' preferences
- Concise — 1-2 sentences between tool calls. Let the visual cards do the talking.
- You describe movies by feel, not data — "gorgeous, melancholic road trip" not "received 7.8 on IMDb"

## Tool Usage Guidelines
- ALWAYS use visual tools — never list movies as plain text
- Use search_movies for any movie search
- Use get_movie_details when discussing a specific film in depth
- Use get_trailer proactively when it would enhance the conversation
- Keep text responses SHORT — let the visual cards speak`;

export const voiceSystemPrompt = `${systemPrompt}

## Voice Mode — IMPORTANT
The user is interacting via voice. All tools display visual cards on screen.
You MUST ALSO describe things verbally since the user may not be looking at the screen.

### After Each Tool
- search_movies: Briefly narrate the top 2-3 results — title, year, one line each
- get_movie_details: Highlight the director, lead actors, and a sentence about the plot
- get_ratings: Speak the score and what it means ("solid 8.1 — critics loved it")
- get_trailer: Let them know the trailer is playing on screen
- compare_movies: Summarize the key differences verbally
- get_recommendations: Read out the top 2-3 picks with brief reasons
- get_person: Mention their most notable roles
- get_streaming_availability: Tell them where it's available
- remember_preference: Just acknowledge verbally ("Got it, noted.")

### Speaking Style
- Conversational — like a friend who loves movies, chatting on the couch
- Describe movies with feeling — "It's this gorgeous, melancholic road trip"
- Keep it concise for voice — shorter than text responses
- Ask one thing at a time — don't overwhelm
- Never read metadata robotically — translate data into human sentences`;

useEffect(() => {
  if (!runnable) return;
  if (voice.isActive) {
    runnable.setSystemPrompt(voiceSystemPrompt);
  } else {
    runnable.setSystemPrompt(systemPrompt);
  }
}, [voice.isActive, runnable]);

import { GloveClient, createRemoteStore } from "glove-react";
import { systemPrompt } from "./system-prompt";
import { storeActions } from "./store-actions";

export const gloveClient = new GloveClient({
  endpoint: "/api/chat",
  systemPrompt,
  createStore: (sessionId) => createRemoteStore(sessionId, storeActions),
});

"use client";

import { useState, useRef, useMemo, useCallback, useEffect } from "react";
import { useGlove } from "glove-react";
import { useGloveVoice } from "glove-react/voice";
import type { TurnMode } from "glove-react/voice";
import { createLolaTools } from "../lib/tools";
import { stt, createTTS, createSileroVAD } from "../lib/voice";
import { systemPrompt, voiceSystemPrompt } from "../lib/system-prompt";
import { VisualArea } from "./visual-area";
import { TranscriptStrip } from "./transcript-strip";
import { VoiceOrb } from "./voice-orb";
import { TextInput } from "./text-input";

interface LolaProps {
  sessionId: string;
  onFirstMessage?: (sessionId: string, text: string) => void;
}

export function Lola({ sessionId, onFirstMessage }: LolaProps) {
  const [turnMode, setTurnMode] = useState<TurnMode>("vad");
  const [isManualRecording, setIsManualRecording] = useState(false);
  const [isProcessing, setIsProcessing] = useState(false);
  const [showTextInput, setShowTextInput] = useState(false);
  const [input, setInput] = useState("");
  const [vadReady, setVadReady] = useState(false);
  const vadRef = useRef<Awaited<ReturnType<typeof createSileroVAD>> | null>(null);

  const MIN_RECORDING_MS = 350;

  // Tools — created once, stable reference
  const tools = useMemo(() => createLolaTools(), []);

  // Glove hook — conversation engine
  const glove = useGlove({ tools, sessionId });
  const {
    runnable, timeline, streamingText, busy,
    slots, sendMessage, renderSlot, renderToolResult,
  } = glove;

  // Silero VAD — async initialization
  useEffect(() => {
    createSileroVAD().then((v) => {
      vadRef.current = v;
      setVadReady(true);
    });
  }, []);

  // Voice pipeline
  const voiceConfig = useMemo(
    () => ({
      stt,
      createTTS,
      vad: vadReady ? vadRef.current ?? undefined : undefined,
      turnMode,
    }),
    [vadReady, turnMode],
  );
  const voice = useGloveVoice({ runnable, voice: voiceConfig });

  // Dynamic system prompt swap
  useEffect(() => {
    if (!runnable) return;
    if (voice.isActive) {
      runnable.setSystemPrompt(voiceSystemPrompt);
    } else {
      runnable.setSystemPrompt(systemPrompt);
    }
  }, [voice.isActive, runnable]);

  // Thinking sound loop
  useEffect(() => {
    if (voice.mode !== "thinking") return;
    const audio = new Audio("/lola-thinking.mp3");
    audio.loop = true;
    audio.play().catch(() => {});
    return () => {
      audio.pause();
      audio.src = "";
    };
  }, [voice.mode]);

  // Last agent text from timeline for transcript strip
  const lastAgentText = useMemo(() => {
    for (let i = timeline.length - 1; i >= 0; i--) {
      const entry = timeline[i];
      if (entry.kind === "agent_text") return entry.text;
    }
    return "";
  }, [timeline]);

  return (
    <div className="lola-screen">
      <VisualArea
        slots={slots}
        timeline={timeline}
        renderSlot={renderSlot}
        renderToolResult={renderToolResult}
        busy={busy}
        onSuggestion={(text) => sendMessage(text)}
      />

      <TranscriptStrip
        text={streamingText || lastAgentText}
        isStreaming={!!streamingText}
      />

      <div className="orb-area">
        {voice.isActive ? (
          <VoiceOrb
            mode={voice.mode}
            transcript={voice.transcript}
            turnMode={turnMode}
            isManualRecording={isManualRecording}
            isProcessing={isProcessing}
            onStop={() => voice.stop()}
            onInterrupt={voice.interrupt}
            onManualRecordStart={() => { /* manual recording logic */ }}
            onManualRecordStop={() => { /* commit recording logic */ }}
          />
        ) : (
          <button
            className="voice-orb voice-orb--idle"
            onClick={() => voice.start()}
          >
            Start Voice
          </button>
        )}

        <TextInput
          visible={showTextInput}
          onToggle={() => setShowTextInput(!showTextInput)}
          input={input}
          setInput={setInput}
          busy={busy}
          onSubmit={(e) => {
            e.preventDefault();
            const text = input.trim();
            if (!text || busy) return;
            setInput("");
            sendMessage(text);
          }}
        />
      </div>
    </div>
  );
}

import { useEffect, useRef, useState } from "react";

interface TranscriptStripProps {
  text: string;
  isStreaming: boolean;
}

const FADE_DELAY_MS = 4000;

export function TranscriptStrip({ text, isStreaming }: TranscriptStripProps) {
  const [isFading, setIsFading] = useState(false);
  const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
  const prevTextRef = useRef(text);

  useEffect(() => {
    // Reset fade when text changes or streaming starts
    if (text !== prevTextRef.current || isStreaming) {
      prevTextRef.current = text;
      setIsFading(false);
      if (timerRef.current) {
        clearTimeout(timerRef.current);
        timerRef.current = null;
      }
    }

    // Start fade timer when not streaming and text exists
    if (!isStreaming && text) {
      timerRef.current = setTimeout(() => {
        setIsFading(true);
        timerRef.current = null;
      }, FADE_DELAY_MS);
    }

    return () => {
      if (timerRef.current) {
        clearTimeout(timerRef.current);
        timerRef.current = null;
      }
    };
  }, [text, isStreaming]);

  if (!text) return null;

  // Show the last ~180 characters, trimmed to a word boundary
  const displayText =
    text.length > 180
      ? "\u2026" + text.slice(text.length - 180).replace(/^\S*\s/, "")
      : text;

  return (
    <div className="transcript-strip" role="status" aria-live="polite">
      <p
        className={`transcript-strip__text ${
          isStreaming ? "transcript-strip__text--streaming" : ""
        } ${isFading ? "transcript-strip__text--fading" : ""}`}
      >
        {displayText}
      </p>
    </div>
  );
}

useEffect(() => {
  if (voice.mode !== "thinking") return;

  const audio = new Audio("/lola-thinking.mp3");
  audio.loop = true;
  audio.play().catch(() => {});

  return () => {
    audio.pause();
    audio.src = "";
  };
}, [voice.mode]);

const MIN_RECORDING_MS = 350;

const commitRecording = useCallback(() => {
  if (!recordingRef.current) return;
  recordingRef.current = false;
  setIsManualRecording(false);

  const elapsed = Date.now() - recordingStartRef.current;

  if (elapsed >= MIN_RECORDING_MS) {
    setIsProcessing(true);
    commitTurnRef.current();
  } else {
    // Delay commit until minimum recording duration
    setIsProcessing(true);
    const remaining = MIN_RECORDING_MS - elapsed;
    pendingCommitRef.current = setTimeout(() => {
      pendingCommitRef.current = null;
      commitTurnRef.current();
    }, remaining);
  }
}, []);

export const VOID = "#0d0d0f";

export const CHARCOAL: Record<number, string> = {
  900: "#1a1a1f",
  800: "#222228",
  700: "#2a2a32",
  600: "#333340",
  500: "#3d3d48",
};

export const AMBER: Record<number, string> = {
  500: "#d4911e",
  400: "#f5a623",
  300: "#f7b84d",
  200: "#fcd88e",
  100: "#fde8b5",
  50: "#fef7e6",
};

export const CREAM = "#faf7f2";
export const CREAM_MUTED = "#a8a4a0";
export const CREAM_DIM = "#706c68";

# From the monorepo root
pnpm install

# Set environment variables in examples/lola/.env.local:
#   OPENROUTER_API_KEY=...
#   TMDB_API_KEY=...
#   ELEVENLABS_API_KEY=...

pnpm --filter glove-lola run dev