Client API

curl -sS "$GATEWAY_URL/v1/chat/completions" \
  -H "Authorization: Bearer $GATEWAY_API_KEY" \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "fast",
    "messages": [{"role": "user", "content": "Reply with gateway-ok"}],
    "max_completion_tokens": 32
  }'

curl -sS "$GATEWAY_URL/v1/audio/transcriptions" \
  -H "Authorization: Bearer $GATEWAY_API_KEY" \
  -F model=transcribe \
  -F language=en \
  -F file=@sample.mp3

curl -sS "$GATEWAY_URL/v1/audio/translations" \
  -H "Authorization: Bearer $GATEWAY_API_KEY" \
  -F model=translate \
  -F file=@sample.mp3

curl -sS "$GATEWAY_URL/v1/audio/speech" \
  -H "Authorization: Bearer $GATEWAY_API_KEY" \
  -H 'Content-Type: application/json' \
  -d '{"model":"tts","input":"Gateway speech is ready.","voice":"default","response_format":"mp3"}' \
  --output speech.mp3

import asyncio
import json
import os

from websockets.asyncio.client import connect


async def main():
    origin = os.environ["GATEWAY_URL"]
    socket_origin = origin.replace("https://", "wss://").replace("http://", "ws://")
    url = f"{socket_origin}/v1/realtime?model=realtime"
    async with connect(
        url,
        additional_headers={
            "Authorization": f"Bearer {os.environ['GATEWAY_API_KEY']}",
            "OpenAI-Safety-Identifier": "end-user-42",
        },
    ) as ws:
        print(json.loads(await ws.recv()))
        await ws.send(json.dumps({
            "type": "session.update",
            "session": {"type": "realtime", "output_modalities": ["audio"]},
        }))


asyncio.run(main())

// Trusted application server. Authenticate your application user before this handler.
const response = await fetch(`${process.env.GATEWAY_URL}/v1/realtime/browser-sessions`, {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.GATEWAY_API_KEY}`,
    "Content-Type": "application/json",
    "OpenAI-Safety-Identifier": hashedApplicationUserId,
  },
  body: JSON.stringify({ model: "realtime" }),
});

const session = await response.json();
// Return this no-store response to the authenticated browser. Do not log or persist it.

const session = await fetch("/api/realtime-session", {
  method: "POST",
  credentials: "same-origin",
}).then((response) => response.json());

const socket = new WebSocket(
  session.websocket.url,
  session.websocket.protocols,
);

socket.addEventListener("open", () => {
  socket.send(JSON.stringify({
    type: "session.update",
    session: { type: "realtime", output_modalities: ["audio"] },
  }));
});

// Trusted application backend. Never put GATEWAY_API_KEY in browser code.
const tokenResponse = await fetch(`${process.env.GATEWAY_URL}/v1/realtime/client_secrets`, {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.GATEWAY_API_KEY}`,
    "Content-Type": "application/json",
    "OpenAI-Safety-Identifier": hashedApplicationUserId,
  },
  body: JSON.stringify({
    expires_after: { anchor: "created_at", seconds: 60 },
    session: {
      type: "realtime",
      model: "realtime",
      audio: { output: { voice: "marin" } },
    },
  }),
});

const bootstrap = await tokenResponse.json();
// Return bootstrap through an authenticated, Cache-Control: no-store response.

const bootstrap = await fetch("/api/realtime-webrtc", {
  method: "POST",
  credentials: "same-origin",
}).then((response) => response.json());

const pc = new RTCPeerConnection();
const audio = document.createElement("audio");
audio.autoplay = true;
pc.ontrack = (event) => { audio.srcObject = event.streams[0]; };

const microphone = await navigator.mediaDevices.getUserMedia({ audio: true });
pc.addTrack(microphone.getAudioTracks()[0]);
const events = pc.createDataChannel("oai-events");

const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
const answer = await fetch(`${GATEWAY_URL}/v1/realtime/calls`, {
  method: "POST",
  headers: {
    Authorization: `Bearer ${bootstrap.value}`,
    "Content-Type": "application/sdp",
  },
  body: offer.sdp,
});
await pc.setRemoteDescription({ type: "answer", sdp: await answer.text() });

events.addEventListener("message", (event) => console.log(JSON.parse(event.data)));

import asyncio
import base64
import json
import os
from pathlib import Path

from websockets.asyncio.client import connect


async def transcribe(pcm_bytes: bytes):
    origin = os.environ["GATEWAY_URL"]
    socket_origin = origin.replace("https://", "wss://").replace("http://", "ws://")
    url = (
        f"{socket_origin}/v1/realtime?model=stt-realtime"
        "&audio_format=pcm_16000&language_code=en&include_timestamps=true"
    )
    async with connect(
        url,
        additional_headers={"Authorization": f"Bearer {os.environ['GATEWAY_API_KEY']}"},
    ) as ws:
        print(json.loads(await ws.recv()))  # session_started
        await ws.send(json.dumps({
            "message_type": "input_audio_chunk",
            "audio_base_64": base64.b64encode(pcm_bytes).decode("ascii"),
            "sample_rate": 16000,
            "commit": True,
        }))
        while True:
            event = json.loads(await ws.recv())
            if event["message_type"].startswith("committed_transcript"):
                return event["text"]


print(asyncio.run(transcribe(Path("sample.pcm").read_bytes())))

import asyncio
import base64
import json
import os

from websockets.asyncio.client import connect


async def synthesize() -> bytes:
    origin = os.environ["GATEWAY_URL"]
    socket_origin = origin.replace("https://", "wss://").replace("http://", "ws://")
    url = (
        f"{socket_origin}/v1/realtime?model=tts-realtime"
        "&voice=default&output_format=pcm"
    )
    chunks = []
    async with connect(
        url,
        additional_headers={"Authorization": f"Bearer {os.environ['GATEWAY_API_KEY']}"},
    ) as ws:
        await ws.send(json.dumps({"text": "Hello from the gateway. ", "flush": True}))
        await ws.send(json.dumps({"text": ""}))
        while True:
            event = json.loads(await ws.recv())
            if event.get("audio"):
                chunks.append(base64.b64decode(event["audio"]))
            if event.get("isFinal") is True:
                return b"".join(chunks)


open("speech.pcm", "wb").write(asyncio.run(synthesize()))

import asyncio
import json
import os

from websockets.asyncio.client import connect


async def talk_to_agent():
    origin = os.environ["GATEWAY_URL"]
    socket_origin = origin.replace("https://", "wss://").replace("http://", "ws://")
    url = f"{socket_origin}/v1/realtime?model=voice-agent"
    async with connect(
        url,
        additional_headers={"Authorization": f"Bearer {os.environ['GATEWAY_API_KEY']}"},
    ) as ws:
        while True:
            event = json.loads(await ws.recv())
            if event["type"] == "ping":
                await ws.send(json.dumps({
                    "type": "pong",
                    "event_id": event["ping_event"]["event_id"],
                }))
            elif event["type"] == "conversation_initiation_metadata":
                await ws.send(json.dumps({
                    "type": "user_message",
                    "text": "What can you help me with?",
                }))
            elif event["type"] == "agent_response":
                return event["agent_response_event"]["agent_response"]


print(asyncio.run(talk_to_agent()))