Realtime Audio

const resp = await fetch("https://api.kymaapi.com/v1/live/sessions", {
  method: "POST",
  headers: {
    "Authorization": `Bearer ${KYMA_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    model: "gemini-2.5-flash-native-audio-preview-12-2025",
    config: {
      speech_config: {
        language_code: "vi",
        voice_config: { prebuilt_voice_config: { voice_name: "Kore" } },
      },
    },
  }),
});
const session = await resp.json();
// session.ws_url, session.session_token, session.heartbeat_url, session.end_url, ...

const ws = new WebSocket(session.ws_url);
ws.binaryType = "arraybuffer";

ws.addEventListener("message", (event) => {
  const frame = JSON.parse(event.data);
  if (frame.setupComplete) {
    // Ready — safe to start sending audio.
    startCaptureLoop();
    return;
  }
  if (frame.serverContent?.modelTurn?.parts) {
    // PCM16 24kHz audio chunk arrived — see Step 4.
    handleAudioChunk(frame.serverContent.modelTurn.parts);
  }
  if (frame.serverContent?.interrupted) {
    // User started speaking over the model — stop any queued playback.
    stopPlayback();
  }
});

const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const audioContext = new AudioContext({ sampleRate: 48000 });
await audioContext.audioWorklet.addModule("/pcm-recorder-worklet.js");
const source = audioContext.createMediaStreamSource(stream);
const worklet = new AudioWorkletNode(audioContext, "pcm-recorder");

worklet.port.onmessage = (event) => {
  // event.data is Int16Array at 16kHz from the worklet.
  const base64 = btoa(String.fromCharCode(...new Uint8Array(event.data.buffer)));
  ws.send(JSON.stringify({
    realtimeInput: {
      mediaChunks: [{ mimeType: "audio/pcm;rate=16000", data: base64 }],
    },
  }));
};

source.connect(worklet);

const playbackContext = new AudioContext({ sampleRate: 24000 });
let nextPlayTime = playbackContext.currentTime;

function handleAudioChunk(parts) {
  for (const part of parts) {
    if (!part.inlineData?.data) continue;
    const bytes = Uint8Array.from(atob(part.inlineData.data), c => c.charCodeAt(0));
    const int16 = new Int16Array(bytes.buffer);
    const float32 = Float32Array.from(int16, v => v / 32768);
    const buffer = playbackContext.createBuffer(1, float32.length, 24000);
    buffer.copyToChannel(float32, 0);
    const source = playbackContext.createBufferSource();
    source.buffer = buffer;
    source.connect(playbackContext.destination);
    nextPlayTime = Math.max(nextPlayTime, playbackContext.currentTime);
    source.start(nextPlayTime);
    nextPlayTime += buffer.duration;
  }
}

const heartbeat = setInterval(() => {
  fetch(session.heartbeat_url, {
    method: "POST",
    headers: { Authorization: `Bearer ${KYMA_API_KEY}` },
  });
}, session.heartbeat_interval_ms); // 30000

// On disconnect:
async function endSession() {
  clearInterval(heartbeat);
  ws.close();
  await fetch(session.end_url, {
    method: "POST",
    headers: { Authorization: `Bearer ${KYMA_API_KEY}` },
  });
}