# Building a realtime voice agent with GPT-Realtime-2.1 and Parallel Search Turbo

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
export async function POST(request: Request) {
  const response = await fetch(
    "https://api.openai.com/v1/realtime/client_secrets",
    {
      method: "POST",
      signal: AbortSignal.timeout(10_000),
      headers: {
        Authorization: "Bearer " + process.env.OPENAI_API_KEY,
        "Content-Type": "application/json",
        "OpenAI-Safety-Identifier": hashUser(request),
      },
      body: JSON.stringify({
        session: {
          type: "realtime",
          model: "gpt-realtime-2.1",
          instructions:
            "Wait for the client session configuration before responding.",
          reasoning: { effort: "low" },
          tools: [],
          tool_choice: "auto",
          audio: {
            input: {
              noise_reduction: { type: "far_field" },
              transcription: {
                language: "en",
                model: "gpt-realtime-whisper",
              },
              turn_detection: {
                type: "semantic_vad",
                eagerness: "medium",
                interrupt_response: true,
                create_response: true,
              },
            },
            output: { voice: "marin" },
          },
        },
      }),
    },
  );

  const data = await response.json();
  return Response.json({ value: data.value }, { status: response.status });
}``` 
export async function POST(request: Request) {
  const response = await fetch(
    "https://api.openai.com/v1/realtime/client_secrets",
    {
      method: "POST",
      signal: AbortSignal.timeout(10_000),
      headers: {
        Authorization: "Bearer " + process.env.OPENAI_API_KEY,
        "Content-Type": "application/json",
        "OpenAI-Safety-Identifier": hashUser(request),
      },
      body: JSON.stringify({
        session: {
          type: "realtime",
          model: "gpt-realtime-2.1",
          instructions:
            "Wait for the client session configuration before responding.",
          reasoning: { effort: "low" },
          tools: [],
          tool_choice: "auto",
          audio: {
            input: {
              noise_reduction: { type: "far_field" },
              transcription: {
                language: "en",
                model: "gpt-realtime-whisper",
              },
              turn_detection: {
                type: "semantic_vad",
                eagerness: "medium",
                interrupt_response: true,
                create_response: true,
              },
            },
            output: { voice: "marin" },
          },
        },
      }),
    },
  );
 
  const data = await response.json();
  return Response.json({ value: data.value }, { status: response.status });
}
``` 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import {
  OpenAIRealtimeWebRTC,
  RealtimeAgent,
  RealtimeSession,
} from "@openai/agents/realtime";

async function connectRealtime() {
  const media = await navigator.mediaDevices.getUserMedia({ audio: true });
  const token = await fetch("/api/realtime-token", { method: "POST" })
    .then((response) => response.json());

  const audio = document.createElement("audio");
  audio.autoplay = true;

  const transport = new OpenAIRealtimeWebRTC({
    mediaStream: media,
    audioElement: audio,
  });

  const agent = new RealtimeAgent({
    name: "Parallel voice assistant",
    instructions: voiceAgentInstructions(),
    tools: [searchWebTool, respondDirectlyTool, waitForUserTool],
  });

  const session = new RealtimeSession(agent, {
    model: "gpt-realtime-2.1",
    transport,
    config: {
      toolChoice: "required",
      parallelToolCalls: false,
      outputModalities: ["text"],
      reasoning: { effort: "low" },
    },
  });

  await session.connect({
    apiKey: token.value,
    model: "gpt-realtime-2.1",
  });

  return session;
}``` 
import {
  OpenAIRealtimeWebRTC,
  RealtimeAgent,
  RealtimeSession,
} from "@openai/agents/realtime";
 
async function connectRealtime() {
  const media = await navigator.mediaDevices.getUserMedia({ audio: true });
  const token = await fetch("/api/realtime-token", { method: "POST" })
    .then((response) => response.json());
 
  const audio = document.createElement("audio");
  audio.autoplay = true;
 
  const transport = new OpenAIRealtimeWebRTC({
    mediaStream: media,
    audioElement: audio,
  });
 
  const agent = new RealtimeAgent({
    name: "Parallel voice assistant",
    instructions: voiceAgentInstructions(),
    tools: [searchWebTool, respondDirectlyTool, waitForUserTool],
  });
 
  const session = new RealtimeSession(agent, {
    model: "gpt-realtime-2.1",
    transport,
    config: {
      toolChoice: "required",
      parallelToolCalls: false,
      outputModalities: ["text"],
      reasoning: { effort: "low" },
    },
  });
 
  await session.connect({
    apiKey: token.value,
    model: "gpt-realtime-2.1",
  });
 
  return session;
}
``` 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
async function searchParallelTurbo(
  question: string,
  generatedQueries: string[],
) {
  const searchQueries = unique([
    question,
    ...generatedQueries,
  ]).slice(0, 3);

  const response = await fetch("https://api.parallel.ai/v1/search", {
    method: "POST",
    signal: AbortSignal.timeout(8_000),
    headers: {
      "Content-Type": "application/json",
      "x-api-key": process.env.PARALLEL_API_KEY!,
    },
    body: JSON.stringify({
      objective: buildSearchObjective(question),
      search_queries: searchQueries,
      mode: "turbo",
      max_chars_total: 10_000,
      advanced_settings: {
        max_results: 20,
        excerpt_settings: { max_chars_per_result: 1_000 },
      },
    }),
  });

  if (!response.ok) throw new Error("Parallel Search failed.");
  return response.json();
}``` 
async function searchParallelTurbo(
  question: string,
  generatedQueries: string[],
) {
  const searchQueries = unique([
    question,
    ...generatedQueries,
  ]).slice(0, 3);
 
  const response = await fetch("https://api.parallel.ai/v1/search", {
    method: "POST",
    signal: AbortSignal.timeout(8_000),
    headers: {
      "Content-Type": "application/json",
      "x-api-key": process.env.PARALLEL_API_KEY!,
    },
    body: JSON.stringify({
      objective: buildSearchObjective(question),
      search_queries: searchQueries,
      mode: "turbo",
      max_chars_total: 10_000,
      advanced_settings: {
        max_results: 20,
        excerpt_settings: { max_chars_per_result: 1_000 },
      },
    }),
  });
 
  if (!response.ok) throw new Error("Parallel Search failed.");
  return response.json();
}
``` 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
session.on(
  "agent_tool_end",
  (_context, _agent, toolDefinition) => {
    if (toolDefinition.name !== "search_web") return;

    session.transport.requestResponse({
      tools: [],
      tool_choice: "none",
      output_modalities: ["audio"],
      max_output_tokens: 800,
      reasoning: { effort: "low" },
      instructions: [
        "Answer directly from the preceding Parallel Search output.",
        "Use one or two complete spoken sentences under 60 words.",
        "Do not mention tools, snippets, JSON, citations, or URLs.",
        "Start with the answer.",
      ].join("\n"),
    });
  },
);``` 
session.on(
  "agent_tool_end",
  (_context, _agent, toolDefinition) => {
    if (toolDefinition.name !== "search_web") return;
 
    session.transport.requestResponse({
      tools: [],
      tool_choice: "none",
      output_modalities: ["audio"],
      max_output_tokens: 800,
      reasoning: { effort: "low" },
      instructions: [
        "Answer directly from the preceding Parallel Search output.",
        "Use one or two complete spoken sentences under 60 words.",
        "Do not mention tools, snippets, JSON, citations, or URLs.",
        "Start with the answer.",
      ].join("\n"),
    });
  },
);
```