diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/.env.example b/contributing/samples/live/live_bidi_streaming_demo_app/.env.example new file mode 100644 index 00000000000..1d2e5ccb480 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/.env.example @@ -0,0 +1,26 @@ +# Copy this file to app/.env and fill in your own values. +# +# cp .env.example app/.env + +# --------------------------------------------------------------------------- +# Option 1: Vertex AI (Google Cloud project + Application Default Credentials) +# --------------------------------------------------------------------------- +GOOGLE_GENAI_USE_VERTEXAI=TRUE +GOOGLE_CLOUD_PROJECT=your-project-id +# Live models are not available in every region. gemini-live-2.5-flash-native-audio +# is served from us-east1. +GOOGLE_CLOUD_LOCATION=us-east1 + +# --------------------------------------------------------------------------- +# Option 2: Gemini API (API key from https://aistudio.google.com/apikey) +# --------------------------------------------------------------------------- +# GOOGLE_GENAI_USE_VERTEXAI=FALSE +# GOOGLE_API_KEY=your-api-key + +# --------------------------------------------------------------------------- +# Model +# --------------------------------------------------------------------------- +# This demo requires a native audio Live model. +# Vertex AI: gemini-live-2.5-flash-native-audio +# Gemini API: gemini-2.5-flash-native-audio-preview-12-2025 +DEMO_AGENT_MODEL=gemini-live-2.5-flash-native-audio diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/README.md b/contributing/samples/live/live_bidi_streaming_demo_app/README.md new file mode 100644 index 00000000000..21e8cdd02bf --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/README.md @@ -0,0 +1,194 @@ +# Live Bidi-Streaming Demo App + +## Overview + +A standalone FastAPI application demonstrating the full ADK bidirectional +streaming lifecycle over a WebSocket, with a browser UI for text, voice, and +camera input. + +Unlike the other samples under `contributing/samples/live/`, this one does not +run under `adk web`. It is a self-contained server that owns its own +`Runner`, `LiveRequestQueue`, and `run_live()` loop, so it shows what an +application built on ADK's streaming primitives actually looks like end to end: + +1. **Application initialization** (once at startup) — create `Agent`, + `SessionService`, and `Runner`. +1. **Session initialization** (per connection) — get or create the `Session`, + build the `RunConfig` and `LiveRequestQueue`, and start `run_live()`. +1. **Bidi-streaming** — concurrent upstream (client → queue) and downstream + (events → client) tasks. +1. **Termination** — close the `LiveRequestQueue` in a `finally` block. + +The bundled web UI includes an event console that renders every raw ADK `Event` +as it arrives, which makes it useful for inspecting Live API behavior. + +This app is the reference implementation for the +[Gemini Live API Toolkit development guide](https://adk.dev/streaming/dev-guide/part1/), +which walks through its source line by line. + +## Sample Inputs + +- `What is the tallest mountain in Japan? Answer in one short sentence.` + + *A plain text turn. The model replies with audio; the transcript appears in + the chat bubble via output transcription.* + +- `What are the top 3 news stories in tech today?` + + *Triggers the `google_search` tool. The event console shows the search + invocation and the grounding metadata.* + +- Click **Start Audio** and say `What is the capital of France?` + + *A voice turn. Input transcription echoes what you said; the reply is spoken + back and transcribed.* + +- Click **📷 Camera**, capture a frame, then ask `What do you see?` + + *An image turn. The frame is sent as a realtime blob before the text turn + opens.* + +## Graph + +```mermaid +graph TD + Browser[browser UI] -->|WebSocket| Server[FastAPI /ws endpoint] + Server -->|upstream task| Queue(LiveRequestQueue) + Queue --> Runner[Runner.run_live] + Runner --> Agent[google_search_agent] + Agent -->|calls| Search(google_search) + Runner -->|downstream task| Server +``` + +## How To + +### Requirements + +- Python 3.10 or higher +- A native audio Live model. This demo always requests the `AUDIO` response + modality, so a half-cascade model will fail with + `Text output is not supported for native audio output model`. + - Vertex AI: `gemini-live-2.5-flash-native-audio` (served from `us-east1`) + - Gemini API: `gemini-2.5-flash-native-audio-preview-12-2025` + +### Setup + +```bash +cd contributing/samples/live/live_bidi_streaming_demo_app +pip install -r requirements.txt +cp .env.example app/.env +``` + +Edit `app/.env` with your credentials. For Vertex AI, authenticate first: + +```bash +gcloud auth application-default login +``` + +> **Note:** `load_dotenv()` does not override variables that are already +> exported in your shell. If your shell exports `GOOGLE_CLOUD_LOCATION` (or any +> other key that also appears in `app/.env`), the shell value wins and the model +> lookup fails with `1008 policy violation: Publisher model ... was not found`. + +### Run + +The server must be started from inside the `app` directory so that Python can +import the `google_search_agent` package: + +```bash +cd app +uvicorn main:app --reload --host 0.0.0.0 --port 8000 +``` + +Then open . + +### WebSocket API + +```text +ws://localhost:8000/ws/{user_id}/{session_id}?proactivity=false&affective_dialog=false +``` + +| Parameter | Kind | Description | +| ------------------ | ----- | ---------------------------------------------------------------- | +| `user_id` | path | Identifies the user | +| `session_id` | path | Identifies the session; reconnecting with the same id resumes it | +| `proactivity` | query | Enable proactive audio (native audio only), default `false` | +| `affective_dialog` | query | Enable affective dialog (native audio only), default `false` | + +Client → server messages: + +```json +{"type": "text", "text": "Your message here"} +``` + +```json +{"type": "image", "data": "", "mimeType": "image/jpeg"} +``` + +Raw binary frames are treated as PCM audio (16 kHz, 16-bit, mono). + +Server → client messages are JSON-serialized ADK `Event` objects. + +### Key techniques + +**Concurrent upstream and downstream tasks.** `asyncio.gather()` runs both +directions at once so the client can keep speaking while the model is +responding. An exception in either task cancels the other, and the `finally` +block always closes the queue: + +```python +try: + await asyncio.gather(upstream_task(), downstream_task()) +except WebSocketDisconnect: + logger.debug("Client disconnected normally") +finally: + live_request_queue.close() +``` + +**Detecting disconnects from the low-level receive loop.** The app calls +Starlette's `websocket.receive()` directly so it can accept both text and binary +frames on one loop. That API reports a client disconnect by *returning* a +message rather than raising, so the loop has to check for it — calling +`receive()` again afterwards raises `RuntimeError`: + +```python +message = await websocket.receive() +if message["type"] == "websocket.disconnect": + raise WebSocketDisconnect(message.get("code", 1000)) +``` + +**Turn-scoped versus realtime input.** Text goes through `send_content()`, which +opens a turn. Audio and image frames go through `send_realtime()`, which streams +continuously and is not turn-scoped — so a captured frame needs to land before +the follow-up text turn opens. + +**Transcription for a native audio model.** Because the response modality is +`AUDIO`, the app enables transcription in both directions to keep a readable +chat transcript: + +```python +run_config = RunConfig( + streaming_mode=StreamingMode.BIDI, + response_modalities=["AUDIO"], + input_audio_transcription=types.AudioTranscriptionConfig(), + output_audio_transcription=types.AudioTranscriptionConfig(), + session_resumption=types.SessionResumptionConfig(), +) +``` + +Transcription arrives as partial events (`finished=false`) followed by one final +event (`finished=true`) carrying the **complete** text. The client replaces the +accumulated text on the final event rather than appending to it — see the +`outputTranscription` handling in `app/static/js/app.js`. + +**Base64url event payloads.** `Event.model_dump_json()` encodes `bytes` as +base64url **without padding**, so the browser normalizes `-`/`_` and re-pads +before decoding audio (`base64ToArray()` in `app/static/js/app.js`). + +## Related Guides + +- [Part 1. Intro to streaming](https://adk.dev/streaming/dev-guide/part1/) — bidirectional streaming fundamentals and the four-phase application lifecycle this app implements. +- [Part 2. Sending messages](https://adk.dev/streaming/dev-guide/part2/) — the `LiveRequestQueue` interface used by the upstream task. +- [Part 3. Event handling](https://adk.dev/streaming/dev-guide/part3/) — the `run_live()` event types the downstream task forwards. +- [Part 4. Run configuration](https://adk.dev/streaming/dev-guide/part4/) — the `RunConfig` options used here, including transcription and session resumption. +- [Part 5. Audio, Images, and Video](https://adk.dev/streaming/dev-guide/part5/) — the audio and video pipelines behind the browser client. diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/google_search_agent/__init__.py b/contributing/samples/live/live_bidi_streaming_demo_app/app/google_search_agent/__init__.py new file mode 100644 index 00000000000..72d2e003ada --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/google_search_agent/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Search Agent package.""" + +from .agent import agent + +__all__ = ["agent"] diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/google_search_agent/agent.py b/contributing/samples/live/live_bidi_streaming_demo_app/app/google_search_agent/agent.py new file mode 100644 index 00000000000..101a327ab89 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/google_search_agent/agent.py @@ -0,0 +1,32 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Search Agent definition for ADK Gemini Live API Toolkit demo.""" + +import os + +from google.adk.agents import Agent +from google.adk.tools import google_search + +# Default models for Live API with native audio support: +# - Gemini Live API: gemini-2.5-flash-native-audio-preview-12-2025 +# - Vertex AI Live API: gemini-live-2.5-flash-native-audio +agent = Agent( + name="google_search_agent", + model=os.getenv( + "DEMO_AGENT_MODEL", "gemini-2.5-flash-native-audio-preview-12-2025" + ), + tools=[google_search], + instruction="You are a helpful assistant that can search the web.", +) diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/main.py b/contributing/samples/live/live_bidi_streaming_demo_app/app/main.py new file mode 100644 index 00000000000..762a76392d4 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/main.py @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""FastAPI application demonstrating ADK Gemini Live API Toolkit with WebSocket.""" + +import asyncio +import base64 +import json +import logging +from pathlib import Path +import warnings + +from dotenv import load_dotenv +from fastapi import FastAPI +from fastapi import WebSocket +from fastapi import WebSocketDisconnect +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles +from google.adk.agents.live_request_queue import LiveRequestQueue +from google.adk.agents.run_config import RunConfig +from google.adk.agents.run_config import StreamingMode +from google.adk.runners import Runner +from google.adk.sessions import InMemorySessionService +from google.genai import types + +# Load environment variables from .env file BEFORE importing agent +load_dotenv(Path(__file__).parent / ".env") + +# Import agent after loading environment variables +# pylint: disable=wrong-import-position +from google_search_agent.agent import agent # noqa: E402 + +# Configure logging +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger("google_adk." + __name__) + +# Suppress Pydantic serialization warnings +warnings.filterwarnings("ignore", category=UserWarning, module="pydantic") + +# Application name constant +APP_NAME = "bidi-demo" + +# ======================================== +# Phase 1: Application Initialization (once at startup) +# ======================================== + +app = FastAPI() + +# Mount static files +static_dir = Path(__file__).parent / "static" +app.mount("/static", StaticFiles(directory=static_dir), name="static") + +# Define your session service +session_service = InMemorySessionService() + +# Define your runner +runner = Runner(app_name=APP_NAME, agent=agent, session_service=session_service) + +# ======================================== +# HTTP Endpoints +# ======================================== + + +@app.get("/") +async def root(): + """Serve the index.html page.""" + return FileResponse(Path(__file__).parent / "static" / "index.html") + + +# ======================================== +# WebSocket Endpoint +# ======================================== + + +@app.websocket("/ws/{user_id}/{session_id}") +async def websocket_endpoint( + websocket: WebSocket, + user_id: str, + session_id: str, + proactivity: bool = False, + affective_dialog: bool = False, +) -> None: + """WebSocket endpoint for bidirectional streaming with ADK. + + Args: + websocket: The WebSocket connection + user_id: User identifier + session_id: Session identifier + proactivity: Enable proactive audio (native audio models only) + affective_dialog: Enable affective dialog (native audio models only) + """ + logger.debug( + f"WebSocket connection request: user_id={user_id}," + f" session_id={session_id}, proactivity={proactivity}," + f" affective_dialog={affective_dialog}" + ) + await websocket.accept() + logger.debug("WebSocket connection accepted") + + # ======================================== + # Phase 2: Session Initialization (once per streaming session) + # ======================================== + + # Native audio models ONLY support the AUDIO response modality, so the + # model speaks its replies and transcription is enabled in both + # directions to keep a readable transcript of the conversation. + # Proactivity and affective dialog are native-audio-only features. + run_config = RunConfig( + streaming_mode=StreamingMode.BIDI, + response_modalities=["AUDIO"], + input_audio_transcription=types.AudioTranscriptionConfig(), + output_audio_transcription=types.AudioTranscriptionConfig(), + session_resumption=types.SessionResumptionConfig(), + proactivity=( + types.ProactivityConfig(proactive_audio=True) if proactivity else None + ), + enable_affective_dialog=affective_dialog if affective_dialog else None, + ) + logger.debug( + f"Model: {agent.model}, using AUDIO response modality, " + f"proactivity={proactivity}, affective_dialog={affective_dialog}" + ) + logger.debug(f"RunConfig created: {run_config}") + + # Get or create session (handles both new sessions and reconnections) + session = await session_service.get_session( + app_name=APP_NAME, user_id=user_id, session_id=session_id + ) + if not session: + await session_service.create_session( + app_name=APP_NAME, user_id=user_id, session_id=session_id + ) + + live_request_queue = LiveRequestQueue() + + # ======================================== + # Phase 3: Active Session (concurrent bidirectional communication) + # ======================================== + + async def upstream_task() -> None: + """Receives messages from WebSocket and sends to LiveRequestQueue.""" + logger.debug("upstream_task started") + while True: + # Receive message from WebSocket (text or binary) + message = await websocket.receive() + + # The low-level receive() reports a client disconnect as a + # message rather than raising, so stop the loop explicitly. + # Calling receive() again after this would raise a RuntimeError. + if message["type"] == "websocket.disconnect": + logger.debug("Client disconnected, stopping upstream_task") + raise WebSocketDisconnect(message.get("code", 1000)) + + # Handle binary frames (audio data) + if "bytes" in message: + audio_data = message["bytes"] + logger.debug(f"Received binary audio chunk: {len(audio_data)} bytes") + + audio_blob = types.Blob( + mime_type="audio/pcm;rate=16000", data=audio_data + ) + live_request_queue.send_realtime(audio_blob) + + # Handle text frames (JSON messages) + elif "text" in message: + text_data = message["text"] + logger.debug(f"Received text message: {text_data[:100]}...") + + json_message = json.loads(text_data) + + # Extract text from JSON and send to LiveRequestQueue + if json_message.get("type") == "text": + logger.debug(f"Sending text content: {json_message['text']}") + content = types.Content(parts=[types.Part(text=json_message["text"])]) + live_request_queue.send_content(content) + + # Handle image data + elif json_message.get("type") == "image": + logger.debug("Received image data") + + # Decode base64 image data + image_data = base64.b64decode(json_message["data"]) + mime_type = json_message.get("mimeType", "image/jpeg") + + logger.debug( + f"Sending image: {len(image_data)} bytes, type: {mime_type}" + ) + + # Send image as blob + image_blob = types.Blob(mime_type=mime_type, data=image_data) + live_request_queue.send_realtime(image_blob) + + async def downstream_task() -> None: + """Receives Events from run_live() and sends to WebSocket.""" + logger.debug("downstream_task started, calling runner.run_live()") + logger.debug( + f"Starting run_live with user_id={user_id}, session_id={session_id}" + ) + async for event in runner.run_live( + user_id=user_id, + session_id=session_id, + live_request_queue=live_request_queue, + run_config=run_config, + ): + event_json = event.model_dump_json(exclude_none=True, by_alias=True) + logger.debug(f"[SERVER] Event: {event_json}") + await websocket.send_text(event_json) + logger.debug("run_live() generator completed") + + # Run both tasks concurrently + # Exceptions from either task will propagate and cancel the other task + try: + logger.debug("Starting asyncio.gather for upstream and downstream tasks") + await asyncio.gather(upstream_task(), downstream_task()) + logger.debug("asyncio.gather completed normally") + except WebSocketDisconnect: + logger.debug("Client disconnected normally") + except Exception as e: + logger.error(f"Unexpected error in streaming tasks: {e}", exc_info=True) + finally: + # ======================================== + # Phase 4: Session Termination + # ======================================== + + # Always close the queue, even if exceptions occurred + logger.debug("Closing live_request_queue") + live_request_queue.close() diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/css/style.css b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/css/style.css new file mode 100644 index 00000000000..c104bd79ed5 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/css/style.css @@ -0,0 +1,915 @@ +/** + * Modern Chat UI Styles for ADK Streaming Demo + */ + +:root { + --primary-color: #4285f4; + --user-bubble-bg: #4285f4; + --agent-bubble-bg: #f1f3f4; + --user-text-color: #ffffff; + --agent-text-color: #202124; + --bg-color: #ffffff; + --border-color: #e0e0e0; + --input-bg: #f8f9fa; + --shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + --shadow-lg: 0 4px 12px rgba(0, 0, 0, 0.15); +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; + background-color: var(--bg-color); + color: #202124; + line-height: 1.6; + height: 100vh; + display: flex; + flex-direction: column; +} + +/* Header */ +header { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + padding: 1.5rem 2rem; + box-shadow: var(--shadow-lg); + position: relative; +} + +h1 { + font-size: 1.5rem; + font-weight: 600; + margin: 0; +} + +.subtitle { + font-size: 0.875rem; + opacity: 0.9; + margin-top: 0.25rem; +} + +/* Header Options (Proactivity, Affective Dialog checkboxes) */ +.header-options { + display: flex; + align-items: center; + gap: 1.25rem; + margin-top: 0.75rem; +} + +.header-checkbox { + display: flex; + align-items: center; + gap: 0.375rem; + font-size: 0.8125rem; + cursor: pointer; + user-select: none; + opacity: 0.9; + transition: opacity 0.2s ease; +} + +.header-checkbox:hover { + opacity: 1; +} + +.header-checkbox input[type="checkbox"] { + width: 16px; + height: 16px; + cursor: pointer; + accent-color: #ffffff; +} + +.header-checkbox span { + white-space: nowrap; +} + +.connection-status { + position: absolute; + top: 1.5rem; + right: 2rem; + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.875rem; +} + +.status-indicator { + width: 8px; + height: 8px; + border-radius: 50%; + background-color: #34a853; +} + +.status-indicator.disconnected { + background-color: #ea4335; +} + +@keyframes pulse { + 0%, 100% { + opacity: 1; + } + 50% { + opacity: 0.5; + } +} + +/* Main Layout: Split view for chat and console */ +.main-layout { + flex: 1; + display: flex; + max-width: 1800px; + width: 100%; + margin: 0 auto; + overflow: hidden; + gap: 0; +} + +/* Main Container: Chat area (2/3 of the layout) */ +.container { + flex: 2; + display: flex; + flex-direction: column; + overflow: hidden; + border-right: 1px solid var(--border-color); +} + +/* Messages Area */ +#messages { + flex: 1; + overflow-y: auto; + padding: 2rem; + display: flex; + flex-direction: column; + gap: 1rem; + background: linear-gradient(to bottom, #f8f9fa 0%, #ffffff 100%); +} + +/* Scroll styling */ +#messages::-webkit-scrollbar { + width: 8px; +} + +#messages::-webkit-scrollbar-track { + background: transparent; +} + +#messages::-webkit-scrollbar-thumb { + background: #dadce0; + border-radius: 4px; +} + +#messages::-webkit-scrollbar-thumb:hover { + background: #bdc1c6; +} + +/* Message Bubbles */ +.message { + display: flex; + margin-bottom: 0.5rem; + animation: slideIn 0.3s ease-out; +} + +@keyframes slideIn { + from { + opacity: 0; + transform: translateY(10px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.message.user { + justify-content: flex-end; +} + +.message.agent { + justify-content: flex-start; +} + +.bubble { + max-width: 70%; + padding: 0.75rem 1rem; + border-radius: 1.25rem; + word-wrap: break-word; + box-shadow: var(--shadow); + position: relative; +} + +.message.user .bubble { + background-color: var(--user-bubble-bg); + color: var(--user-text-color); + border-bottom-right-radius: 0.25rem; +} + +.message.agent .bubble { + background-color: var(--agent-bubble-bg); + color: var(--agent-text-color); + border-bottom-left-radius: 0.25rem; +} + +.bubble-text { + margin: 0; + line-height: 1.5; +} + +/* Interrupted message styling */ +.message.interrupted .bubble { + opacity: 0.6; + background-color: #e8eaed; + border-left: 3px solid #f4b400; +} + +.message.interrupted .bubble::after { + content: 'interrupted'; + display: block; + font-size: 0.75rem; + color: #5f6368; + font-style: italic; + margin-top: 0.25rem; +} + +/* Transcription message styling */ +.message.transcription.user .bubble { + opacity: 0.9; + border: 1px solid rgba(255, 255, 255, 0.3); +} + +.message.transcription.user .bubble::before { + content: '🎤'; + opacity: 0.8; + margin-right: 0.25rem; +} + +/* Typing indicator */ +.typing-indicator { + display: inline-block; + margin-left: 0.25rem; + color: #5f6368; +} + +.typing-indicator::after { + content: '...'; + animation: ellipsis 1.5s infinite; +} + +@keyframes ellipsis { + 0%, 20% { + content: '.'; + } + 40% { + content: '..'; + } + 60%, 100% { + content: '...'; + } +} + +/* Image bubble styling */ +.bubble.image-bubble { + padding: 0.25rem; + max-width: 80%; +} + +.bubble-image { + max-width: 100%; + max-height: 300px; + width: auto; + height: auto; + border-radius: 0.75rem; + display: block; + object-fit: contain; +} + +/* Input Form */ +.input-container { + border-top: 1px solid var(--border-color); + background-color: white; + padding: 1.5rem 2rem; + box-shadow: 0 -2px 8px rgba(0, 0, 0, 0.05); +} + +#messageForm { + display: flex; + gap: 1rem; + align-items: center; +} + +.input-wrapper { + flex: 1; + display: flex; + gap: 0.75rem; +} + +#message { + flex: 1; + padding: 0.875rem 1rem; + border: 1px solid var(--border-color); + border-radius: 1.5rem; + font-size: 1rem; + font-family: inherit; + background-color: var(--input-bg); + transition: all 0.2s ease; + outline: none; +} + +#message:focus { + border-color: var(--primary-color); + background-color: white; + box-shadow: 0 0 0 3px rgba(66, 133, 244, 0.1); +} + +/* Buttons */ +button { + padding: 0.875rem 1.5rem; + border: none; + border-radius: 1.5rem; + font-size: 0.9375rem; + font-weight: 500; + cursor: pointer; + transition: all 0.2s ease; + font-family: inherit; + white-space: nowrap; +} + +#sendButton { + background-color: var(--primary-color); + color: white; +} + +#sendButton:hover:not(:disabled) { + background-color: #3367d6; + box-shadow: var(--shadow); + transform: translateY(-1px); +} + +#sendButton:disabled { + background-color: #dadce0; + color: #80868b; + cursor: not-allowed; + opacity: 0.6; +} + +#startAudioButton { + background-color: #34a853; + color: white; +} + +#startAudioButton:hover:not(:disabled) { + background-color: #2d8e47; + box-shadow: var(--shadow); + transform: translateY(-1px); +} + +#startAudioButton:disabled { + background-color: #dadce0; + color: #80868b; + cursor: not-allowed; + opacity: 0.6; +} + +#cameraButton { + background-color: #ea4335; + color: white; +} + +#cameraButton:hover:not(:disabled) { + background-color: #d33426; + box-shadow: var(--shadow); + transform: translateY(-1px); +} + +#cameraButton:disabled { + background-color: #dadce0; + color: #80868b; + cursor: not-allowed; + opacity: 0.6; +} + +/* System Messages */ +.system-message { + text-align: center; + color: #5f6368; + font-size: 0.875rem; + padding: 0.5rem; + margin: 1rem 0; + font-style: italic; +} + +/* Console Panel (1/3 of the layout) */ +.console-panel { + flex: 1; + display: flex; + flex-direction: column; + background-color: #1e1e1e; + color: #d4d4d4; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Consolas', 'source-code-pro', monospace; + overflow: hidden; +} + +.console-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background-color: #2d2d2d; + border-bottom: 1px solid #3e3e3e; +} + +.console-header h2 { + font-size: 0.875rem; + font-weight: 600; + margin: 0; + color: #cccccc; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.console-controls { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.console-checkbox { + display: flex; + align-items: center; + gap: 0.375rem; + font-size: 0.75rem; + color: #999999; + cursor: pointer; + user-select: none; +} + +.console-checkbox input[type="checkbox"] { + width: 14px; + height: 14px; + cursor: pointer; + accent-color: #4285f4; +} + +.console-checkbox span { + white-space: nowrap; +} + +.console-clear-btn { + padding: 0.375rem 0.75rem; + font-size: 0.75rem; + background-color: #3e3e3e; + color: #cccccc; + border: 1px solid #4e4e4e; + border-radius: 0.25rem; + cursor: pointer; + transition: all 0.2s ease; +} + +.console-clear-btn:hover { + background-color: #4e4e4e; + border-color: #5e5e5e; +} + +.console-content { + flex: 1; + overflow-y: auto; + padding: 0.75rem; + font-size: 0.75rem; + line-height: 1.5; +} + +/* Console entry */ +.console-entry { + margin-bottom: 0.75rem; + padding: 0.5rem; + border-left: 3px solid transparent; + background-color: rgba(255, 255, 255, 0.06); + border-radius: 0.25rem; + transition: background-color 0.2s ease; +} + +.console-entry.outgoing { + border-left-color: #4285f4; +} + +.console-entry.incoming { + border-left-color: #34a853; +} + +.console-entry.error { + border-left-color: #ea4335; + background-color: rgba(234, 67, 53, 0.15); +} + +/* Expandable console entries */ +.console-entry.expandable { + cursor: pointer; +} + +.console-entry.expandable:hover { + background-color: rgba(255, 255, 255, 0.10); +} + +.console-entry.expanded { + background-color: rgba(255, 255, 255, 0.08); +} + +.console-entry-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.375rem; +} + +.console-entry-left { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.console-entry-emoji { + font-size: 0.9rem; + line-height: 1; + display: inline-block; + user-select: none; + min-width: 16px; + text-align: center; +} + +.console-expand-icon { + font-size: 0.6rem; + color: #858585; + width: 12px; + display: inline-block; + transition: transform 0.2s ease; + user-select: none; +} + +.console-entry-type { + font-weight: 600; + font-size: 0.7rem; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.console-entry.outgoing .console-entry-type { + color: #4285f4; +} + +.console-entry.incoming .console-entry-type { + color: #34a853; +} + +.console-entry.error .console-entry-type { + color: #ea4335; +} + +.console-entry-author { + font-size: 0.65rem; + font-weight: 500; + padding: 0.125rem 0.375rem; + border-radius: 0.25rem; + text-transform: lowercase; + letter-spacing: 0.3px; + border: 1px solid; +} + +/* Default style (for agents) */ +.console-entry-author { + background-color: rgba(156, 220, 254, 0.15); + color: #9cdcfe; + border-color: rgba(156, 220, 254, 0.3); +} + +/* User author badge */ +.console-entry-author[data-author="user"] { + background-color: rgba(66, 133, 244, 0.2); + color: #80b3ff; + border-color: rgba(66, 133, 244, 0.4); +} + +/* System author badge */ +.console-entry-author[data-author="system"] { + background-color: rgba(133, 133, 133, 0.2); + color: #b0b0b0; + border-color: rgba(133, 133, 133, 0.3); +} + +.console-entry-timestamp { + color: #858585; + font-size: 0.65rem; +} + +.console-entry-content { + color: #d4d4d4; + white-space: pre-wrap; + word-break: break-word; + font-size: 0.7rem; + line-height: 1.4; + padding-left: 2.5rem; /* Indent to align with header after emoji and icon */ +} + +.console-entry-content:empty { + display: none; +} + +/* Style for quoted text in summaries (transcriptions, text responses) */ +.console-entry-content::first-line { + color: #e0e0e0; +} + +.console-entry-json { + background-color: #252526; + padding: 0.5rem; + border-radius: 0.25rem; + margin-top: 0.5rem; + overflow-x: auto; + max-height: 400px; + overflow-y: auto; + transition: all 0.3s ease; +} + +.console-entry-json.collapsed { + display: none; +} + +.console-entry-json pre { + margin: 0; + color: #9cdcfe; +} + +/* Highlight key fields in JSON */ +.json-key { + color: #9cdcfe; +} + +.json-string { + color: #ce9178; +} + +.json-number { + color: #b5cea8; +} + +.json-boolean { + color: #569cd6; +} + +.json-null { + color: #569cd6; +} + +/* Console scrollbar */ +.console-content::-webkit-scrollbar { + width: 8px; +} + +.console-content::-webkit-scrollbar-track { + background: #1e1e1e; +} + +.console-content::-webkit-scrollbar-thumb { + background: #3e3e3e; + border-radius: 4px; +} + +.console-content::-webkit-scrollbar-thumb:hover { + background: #4e4e4e; +} + +/* JSON scrollbar */ +.console-entry-json::-webkit-scrollbar { + width: 6px; + height: 6px; +} + +.console-entry-json::-webkit-scrollbar-track { + background: #1e1e1e; +} + +.console-entry-json::-webkit-scrollbar-thumb { + background: #3e3e3e; + border-radius: 3px; +} + +.console-entry-json::-webkit-scrollbar-thumb:hover { + background: #4e4e4e; +} + +/* Responsive Design */ +@media (max-width: 768px) { + header { + padding: 1rem 1.5rem; + } + + h1 { + font-size: 1.25rem; + } + + .connection-status { + position: static; + margin-top: 0.5rem; + } + + /* Stack console panel below chat on mobile */ + .main-layout { + flex-direction: column; + } + + .console-panel { + max-height: 300px; + border-top: 1px solid var(--border-color); + } + + .container { + border-right: none; + } + + #messages { + padding: 1rem; + } + + .bubble { + max-width: 85%; + } + + .input-container { + padding: 1rem; + } + + #messageForm { + flex-direction: column; + gap: 0.75rem; + } + + .input-wrapper { + width: 100%; + flex-direction: column; + } + + button { + width: 100%; + } +} + +/* Loading state */ +.loading { + display: inline-block; + width: 20px; + height: 20px; + border: 3px solid rgba(255, 255, 255, 0.3); + border-radius: 50%; + border-top-color: white; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +/* Camera Preview Modal */ +.modal { + display: none; + position: fixed; + z-index: 1000; + left: 0; + top: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.7); + backdrop-filter: blur(4px); +} + +.modal.show { + display: flex; + align-items: center; + justify-content: center; +} + +.modal-content { + background-color: white; + border-radius: 1rem; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.3); + max-width: 90%; + max-height: 90%; + width: 640px; + display: flex; + flex-direction: column; + animation: modalSlideIn 0.3s ease-out; +} + +@keyframes modalSlideIn { + from { + opacity: 0; + transform: translateY(-20px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.modal-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1.5rem; + border-bottom: 1px solid var(--border-color); +} + +.modal-header h3 { + margin: 0; + font-size: 1.25rem; + font-weight: 600; + color: #202124; +} + +.close-btn { + background: none; + border: none; + font-size: 2rem; + line-height: 1; + color: #5f6368; + cursor: pointer; + padding: 0; + width: 32px; + height: 32px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 50%; + transition: all 0.2s ease; +} + +.close-btn:hover { + background-color: #f1f3f4; + color: #202124; +} + +.modal-body { + padding: 1.5rem; + flex: 1; + display: flex; + align-items: center; + justify-content: center; + background-color: #000; + border-radius: 0 0 1rem 1rem; +} + +#cameraPreview { + width: 100%; + max-height: 480px; + border-radius: 0.5rem; + object-fit: contain; +} + +.modal-footer { + padding: 1.5rem; + border-top: 1px solid var(--border-color); + display: flex; + gap: 1rem; + justify-content: flex-end; + background-color: white; + border-radius: 0 0 1rem 1rem; +} + +.btn-primary { + background-color: var(--primary-color); + color: white; + padding: 0.875rem 1.5rem; + border: none; + border-radius: 1.5rem; + font-size: 0.9375rem; + font-weight: 500; + cursor: pointer; + transition: all 0.2s ease; + font-family: inherit; +} + +.btn-primary:hover { + background-color: #3367d6; + box-shadow: var(--shadow); + transform: translateY(-1px); +} + +.btn-secondary { + background-color: white; + color: #5f6368; + padding: 0.875rem 1.5rem; + border: 1px solid var(--border-color); + border-radius: 1.5rem; + font-size: 0.9375rem; + font-weight: 500; + cursor: pointer; + transition: all 0.2s ease; + font-family: inherit; +} + +.btn-secondary:hover { + background-color: #f8f9fa; + border-color: #bdc1c6; +} diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/index.html b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/index.html new file mode 100644 index 00000000000..ed71cebebd6 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/index.html @@ -0,0 +1,85 @@ + + + + + + ADK Gemini Live API Toolkit Demo + + + + + +
+

ADK Gemini Live API Toolkit Demo

+
Real-time bidirectional streaming with Google ADK
+
+ + +
+
+ + Connecting... +
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+ +
+
+

Event Console

+
+ + +
+
+
+
+
+ + + + + diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/app.js b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/app.js new file mode 100644 index 00000000000..e9a3ad1c8eb --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/app.js @@ -0,0 +1,1003 @@ +/** + * app.js: JS code for the ADK Gemini Live API Toolkit demo app. + */ + +/** + * WebSocket handling + */ + +// Connect the server with a WebSocket connection +const userId = "demo-user"; +const sessionId = "demo-session-" + Math.random().toString(36).substring(7); +let websocket = null; +let is_audio = false; + +// Get checkbox elements for RunConfig options +const enableProactivityCheckbox = document.getElementById("enableProactivity"); +const enableAffectiveDialogCheckbox = document.getElementById("enableAffectiveDialog"); + +// Reconnect WebSocket when RunConfig options change +function handleRunConfigChange() { + if (websocket && websocket.readyState === WebSocket.OPEN) { + addSystemMessage("Reconnecting with updated settings..."); + addConsoleEntry('outgoing', 'Reconnecting due to settings change', { + proactivity: enableProactivityCheckbox.checked, + affective_dialog: enableAffectiveDialogCheckbox.checked + }, '🔄', 'system'); + websocket.close(); + // connectWebsocket() will be called by onclose handler after delay + } +} + +// Add change listeners to RunConfig checkboxes +enableProactivityCheckbox.addEventListener("change", handleRunConfigChange); +enableAffectiveDialogCheckbox.addEventListener("change", handleRunConfigChange); + +// Build WebSocket URL with RunConfig options as query parameters +function getWebSocketUrl() { + // Use wss:// for HTTPS pages, ws:// for HTTP (localhost development) + const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; + const baseUrl = wsProtocol + "//" + window.location.host + "/ws/" + userId + "/" + sessionId; + const params = new URLSearchParams(); + + // Add proactivity option if checked + if (enableProactivityCheckbox && enableProactivityCheckbox.checked) { + params.append("proactivity", "true"); + } + + // Add affective dialog option if checked + if (enableAffectiveDialogCheckbox && enableAffectiveDialogCheckbox.checked) { + params.append("affective_dialog", "true"); + } + + const queryString = params.toString(); + return queryString ? baseUrl + "?" + queryString : baseUrl; +} + +// Get DOM elements +const messageForm = document.getElementById("messageForm"); +const messageInput = document.getElementById("message"); +const messagesDiv = document.getElementById("messages"); +const statusIndicator = document.getElementById("statusIndicator"); +const statusText = document.getElementById("statusText"); +const consoleContent = document.getElementById("consoleContent"); +const clearConsoleBtn = document.getElementById("clearConsole"); +const showAudioEventsCheckbox = document.getElementById("showAudioEvents"); +let currentMessageId = null; +let currentBubbleElement = null; +let currentInputTranscriptionId = null; +let currentInputTranscriptionElement = null; +let currentOutputTranscriptionId = null; +let currentOutputTranscriptionElement = null; +let inputTranscriptionFinished = false; // Track if input transcription is complete for this turn +let hasOutputTranscriptionInTurn = false; // Track if output transcription delivered the response + +// Helper function to clean spaces between CJK characters +// Removes spaces between Japanese/Chinese/Korean characters while preserving spaces around Latin text +function cleanCJKSpaces(text) { + // CJK Unicode ranges: Hiragana, Katakana, Kanji, CJK Unified Ideographs, Fullwidth forms + const cjkPattern = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9faf\uff00-\uffef]/; + + // Remove spaces between two CJK characters + return text.replace(/(\S)\s+(?=\S)/g, (match, char1) => { + // Get the character after the space(s) + const nextCharMatch = text.match(new RegExp(char1 + '\\s+(.)', 'g')); + if (nextCharMatch && nextCharMatch.length > 0) { + const char2 = nextCharMatch[0].slice(-1); + // If both characters are CJK, remove the space + if (cjkPattern.test(char1) && cjkPattern.test(char2)) { + return char1; + } + } + return match; + }); +} + +// Console logging functionality +function formatTimestamp() { + const now = new Date(); + return now.toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit', fractionalSecondDigits: 3 }); +} + +function addConsoleEntry(type, content, data = null, emoji = null, author = null, isAudio = false) { + // Skip audio events if checkbox is unchecked + if (isAudio && !showAudioEventsCheckbox.checked) { + return; + } + + const entry = document.createElement("div"); + entry.className = `console-entry ${type}`; + + const header = document.createElement("div"); + header.className = "console-entry-header"; + + const leftSection = document.createElement("div"); + leftSection.className = "console-entry-left"; + + // Add emoji icon if provided + if (emoji) { + const emojiIcon = document.createElement("span"); + emojiIcon.className = "console-entry-emoji"; + emojiIcon.textContent = emoji; + leftSection.appendChild(emojiIcon); + } + + // Add expand/collapse icon + const expandIcon = document.createElement("span"); + expandIcon.className = "console-expand-icon"; + expandIcon.textContent = data ? "▶" : ""; + + const typeLabel = document.createElement("span"); + typeLabel.className = "console-entry-type"; + typeLabel.textContent = type === 'outgoing' ? '↑ Upstream' : type === 'incoming' ? '↓ Downstream' : '⚠ Error'; + + leftSection.appendChild(expandIcon); + leftSection.appendChild(typeLabel); + + // Add author badge if provided + if (author) { + const authorBadge = document.createElement("span"); + authorBadge.className = "console-entry-author"; + authorBadge.textContent = author; + authorBadge.setAttribute('data-author', author); + leftSection.appendChild(authorBadge); + } + + const timestamp = document.createElement("span"); + timestamp.className = "console-entry-timestamp"; + timestamp.textContent = formatTimestamp(); + + header.appendChild(leftSection); + header.appendChild(timestamp); + + const contentDiv = document.createElement("div"); + contentDiv.className = "console-entry-content"; + contentDiv.textContent = content; + + entry.appendChild(header); + entry.appendChild(contentDiv); + + // JSON details (hidden by default) + let jsonDiv = null; + if (data) { + jsonDiv = document.createElement("div"); + jsonDiv.className = "console-entry-json collapsed"; + const pre = document.createElement("pre"); + pre.textContent = JSON.stringify(data, null, 2); + jsonDiv.appendChild(pre); + entry.appendChild(jsonDiv); + + // Make entry clickable if it has data + entry.classList.add("expandable"); + + // Toggle expand/collapse on click + entry.addEventListener("click", () => { + const isExpanded = !jsonDiv.classList.contains("collapsed"); + + if (isExpanded) { + // Collapse + jsonDiv.classList.add("collapsed"); + expandIcon.textContent = "▶"; + entry.classList.remove("expanded"); + } else { + // Expand + jsonDiv.classList.remove("collapsed"); + expandIcon.textContent = "▼"; + entry.classList.add("expanded"); + } + }); + } + + consoleContent.appendChild(entry); + consoleContent.scrollTop = consoleContent.scrollHeight; +} + +function clearConsole() { + consoleContent.innerHTML = ''; +} + +// Clear console button handler +clearConsoleBtn.addEventListener('click', clearConsole); + +// Update connection status UI +function updateConnectionStatus(connected) { + if (connected) { + statusIndicator.classList.remove("disconnected"); + statusText.textContent = "Connected"; + } else { + statusIndicator.classList.add("disconnected"); + statusText.textContent = "Disconnected"; + } +} + +// Create a message bubble element +function createMessageBubble(text, isUser, isPartial = false) { + const messageDiv = document.createElement("div"); + messageDiv.className = `message ${isUser ? "user" : "agent"}`; + + const bubbleDiv = document.createElement("div"); + bubbleDiv.className = "bubble"; + + const textP = document.createElement("p"); + textP.className = "bubble-text"; + textP.textContent = text; + + // Add typing indicator for partial messages + if (isPartial && !isUser) { + const typingSpan = document.createElement("span"); + typingSpan.className = "typing-indicator"; + textP.appendChild(typingSpan); + } + + bubbleDiv.appendChild(textP); + messageDiv.appendChild(bubbleDiv); + + return messageDiv; +} + +// Create an image message bubble element +function createImageBubble(imageDataUrl, isUser) { + const messageDiv = document.createElement("div"); + messageDiv.className = `message ${isUser ? "user" : "agent"}`; + + const bubbleDiv = document.createElement("div"); + bubbleDiv.className = "bubble image-bubble"; + + const img = document.createElement("img"); + img.src = imageDataUrl; + img.className = "bubble-image"; + img.alt = "Captured image"; + + bubbleDiv.appendChild(img); + messageDiv.appendChild(bubbleDiv); + + return messageDiv; +} + +// Update existing message bubble text +function updateMessageBubble(element, text, isPartial = false) { + const textElement = element.querySelector(".bubble-text"); + + // Remove existing typing indicator + const existingIndicator = textElement.querySelector(".typing-indicator"); + if (existingIndicator) { + existingIndicator.remove(); + } + + textElement.textContent = text; + + // Add typing indicator for partial messages + if (isPartial) { + const typingSpan = document.createElement("span"); + typingSpan.className = "typing-indicator"; + textElement.appendChild(typingSpan); + } +} + +// Add a system message +function addSystemMessage(text) { + const messageDiv = document.createElement("div"); + messageDiv.className = "system-message"; + messageDiv.textContent = text; + messagesDiv.appendChild(messageDiv); + scrollToBottom(); +} + +// Scroll to bottom of messages +function scrollToBottom() { + messagesDiv.scrollTop = messagesDiv.scrollHeight; +} + +// Sanitize event data for console display (replace large audio data with summary) +function sanitizeEventForDisplay(event) { + // Deep clone the event object + const sanitized = JSON.parse(JSON.stringify(event)); + + // Check for audio data in content.parts + if (sanitized.content && sanitized.content.parts) { + sanitized.content.parts = sanitized.content.parts.map(part => { + if (part.inlineData && part.inlineData.data) { + // Calculate byte size (base64 string length / 4 * 3, roughly) + const byteSize = Math.floor(part.inlineData.data.length * 0.75); + return { + ...part, + inlineData: { + ...part.inlineData, + data: `(${byteSize.toLocaleString()} bytes)` + } + }; + } + return part; + }); + } + + return sanitized; +} + +// WebSocket handlers +function connectWebsocket() { + // Connect websocket + const ws_url = getWebSocketUrl(); + websocket = new WebSocket(ws_url); + + // Handle connection open + websocket.onopen = function () { + console.log("WebSocket connection opened."); + updateConnectionStatus(true); + addSystemMessage("Connected to ADK streaming server"); + + // Log to console + addConsoleEntry('incoming', 'WebSocket Connected', { + userId: userId, + sessionId: sessionId, + url: ws_url + }, '🔌', 'system'); + + // Enable the Send button + document.getElementById("sendButton").disabled = false; + addSubmitHandler(); + }; + + // Handle incoming messages + websocket.onmessage = function (event) { + // Parse the incoming ADK Event + const adkEvent = JSON.parse(event.data); + console.log("[AGENT TO CLIENT] ", adkEvent); + + // Log to console panel + let eventSummary = 'Event'; + let eventEmoji = '📨'; // Default emoji + const author = adkEvent.author || 'system'; + + if (adkEvent.turnComplete) { + eventSummary = 'Turn Complete'; + eventEmoji = '✅'; + } else if (adkEvent.interrupted) { + eventSummary = 'Interrupted'; + eventEmoji = '⏸️'; + } else if (adkEvent.inputTranscription) { + // Show transcription text in summary + const transcriptionText = adkEvent.inputTranscription.text || ''; + const truncated = transcriptionText.length > 60 + ? transcriptionText.substring(0, 60) + '...' + : transcriptionText; + eventSummary = `Input Transcription: "${truncated}"`; + eventEmoji = '📝'; + } else if (adkEvent.outputTranscription) { + // Show transcription text in summary + const transcriptionText = adkEvent.outputTranscription.text || ''; + const truncated = transcriptionText.length > 60 + ? transcriptionText.substring(0, 60) + '...' + : transcriptionText; + eventSummary = `Output Transcription: "${truncated}"`; + eventEmoji = '📝'; + } else if (adkEvent.usageMetadata) { + // Show token usage information + const usage = adkEvent.usageMetadata; + const promptTokens = usage.promptTokenCount || 0; + const responseTokens = usage.candidatesTokenCount || 0; + const totalTokens = usage.totalTokenCount || 0; + eventSummary = `Token Usage: ${totalTokens.toLocaleString()} total (${promptTokens.toLocaleString()} prompt + ${responseTokens.toLocaleString()} response)`; + eventEmoji = '📊'; + } else if (adkEvent.content && adkEvent.content.parts) { + const hasText = adkEvent.content.parts.some(p => p.text); + const hasAudio = adkEvent.content.parts.some(p => p.inlineData); + const hasExecutableCode = adkEvent.content.parts.some(p => p.executableCode); + const hasCodeExecutionResult = adkEvent.content.parts.some(p => p.codeExecutionResult); + + if (hasExecutableCode) { + // Show executable code + const codePart = adkEvent.content.parts.find(p => p.executableCode); + if (codePart && codePart.executableCode) { + const code = codePart.executableCode.code || ''; + const language = codePart.executableCode.language || 'unknown'; + const truncated = code.length > 60 + ? code.substring(0, 60).replace(/\n/g, ' ') + '...' + : code.replace(/\n/g, ' '); + eventSummary = `Executable Code (${language}): ${truncated}`; + eventEmoji = '💻'; + } + } + + if (hasCodeExecutionResult) { + // Show code execution result + const resultPart = adkEvent.content.parts.find(p => p.codeExecutionResult); + if (resultPart && resultPart.codeExecutionResult) { + const outcome = resultPart.codeExecutionResult.outcome || 'UNKNOWN'; + const output = resultPart.codeExecutionResult.output || ''; + const truncatedOutput = output.length > 60 + ? output.substring(0, 60).replace(/\n/g, ' ') + '...' + : output.replace(/\n/g, ' '); + eventSummary = `Code Execution Result (${outcome}): ${truncatedOutput}`; + eventEmoji = outcome === 'OUTCOME_OK' ? '✅' : '❌'; + } + } + + if (hasText) { + // Show text preview in summary + const textPart = adkEvent.content.parts.find(p => p.text); + if (textPart && textPart.text) { + const text = textPart.text; + const truncated = text.length > 80 + ? text.substring(0, 80) + '...' + : text; + eventSummary = `Text: "${truncated}"`; + eventEmoji = '💭'; + } else { + eventSummary = 'Text Response'; + eventEmoji = '💭'; + } + } + + if (hasAudio) { + // Extract audio info for summary + const audioPart = adkEvent.content.parts.find(p => p.inlineData); + if (audioPart && audioPart.inlineData) { + const mimeType = audioPart.inlineData.mimeType || 'unknown'; + const dataLength = audioPart.inlineData.data ? audioPart.inlineData.data.length : 0; + // Base64 string length / 4 * 3 gives approximate bytes + const byteSize = Math.floor(dataLength * 0.75); + eventSummary = `Audio Response: ${mimeType} (${byteSize.toLocaleString()} bytes)`; + eventEmoji = '🔊'; + } else { + eventSummary = 'Audio Response'; + eventEmoji = '🔊'; + } + + // Log audio event with isAudio flag (filtered by checkbox) + const sanitizedEvent = sanitizeEventForDisplay(adkEvent); + addConsoleEntry('incoming', eventSummary, sanitizedEvent, eventEmoji, author, true); + } + } + + // Create a sanitized version for console display (replace large audio data with summary) + // Skip if already logged as audio event above + const isAudioOnlyEvent = adkEvent.content && adkEvent.content.parts && + adkEvent.content.parts.some(p => p.inlineData) && + !adkEvent.content.parts.some(p => p.text); + if (!isAudioOnlyEvent) { + const sanitizedEvent = sanitizeEventForDisplay(adkEvent); + addConsoleEntry('incoming', eventSummary, sanitizedEvent, eventEmoji, author); + } + + // Handle turn complete event + if (adkEvent.turnComplete === true) { + // Remove typing indicator from current message + if (currentBubbleElement) { + const textElement = currentBubbleElement.querySelector(".bubble-text"); + const typingIndicator = textElement.querySelector(".typing-indicator"); + if (typingIndicator) { + typingIndicator.remove(); + } + } + // Remove typing indicator from current output transcription + if (currentOutputTranscriptionElement) { + const textElement = currentOutputTranscriptionElement.querySelector(".bubble-text"); + const typingIndicator = textElement.querySelector(".typing-indicator"); + if (typingIndicator) { + typingIndicator.remove(); + } + } + currentMessageId = null; + currentBubbleElement = null; + currentOutputTranscriptionId = null; + currentOutputTranscriptionElement = null; + inputTranscriptionFinished = false; // Reset for next turn + hasOutputTranscriptionInTurn = false; // Reset for next turn + return; + } + + // Handle interrupted event + if (adkEvent.interrupted === true) { + // Stop audio playback if it's playing + if (audioPlayerNode) { + audioPlayerNode.port.postMessage({ command: "endOfAudio" }); + } + + // Keep the partial message but mark it as interrupted + if (currentBubbleElement) { + const textElement = currentBubbleElement.querySelector(".bubble-text"); + + // Remove typing indicator + const typingIndicator = textElement.querySelector(".typing-indicator"); + if (typingIndicator) { + typingIndicator.remove(); + } + + // Add interrupted marker + currentBubbleElement.classList.add("interrupted"); + } + + // Keep the partial output transcription but mark it as interrupted + if (currentOutputTranscriptionElement) { + const textElement = currentOutputTranscriptionElement.querySelector(".bubble-text"); + + // Remove typing indicator + const typingIndicator = textElement.querySelector(".typing-indicator"); + if (typingIndicator) { + typingIndicator.remove(); + } + + // Add interrupted marker + currentOutputTranscriptionElement.classList.add("interrupted"); + } + + // Reset state so new content creates a new bubble + currentMessageId = null; + currentBubbleElement = null; + currentOutputTranscriptionId = null; + currentOutputTranscriptionElement = null; + inputTranscriptionFinished = false; // Reset for next turn + hasOutputTranscriptionInTurn = false; // Reset for next turn + return; + } + + // Handle input transcription (user's spoken words) + if (adkEvent.inputTranscription && adkEvent.inputTranscription.text) { + const transcriptionText = adkEvent.inputTranscription.text; + const isFinished = adkEvent.inputTranscription.finished; + + if (transcriptionText) { + // Ignore late-arriving transcriptions after we've finished for this turn + if (inputTranscriptionFinished) { + return; + } + + if (currentInputTranscriptionId == null) { + // Create new transcription bubble + currentInputTranscriptionId = Math.random().toString(36).substring(7); + // Clean spaces between CJK characters + const cleanedText = cleanCJKSpaces(transcriptionText); + currentInputTranscriptionElement = createMessageBubble(cleanedText, true, !isFinished); + currentInputTranscriptionElement.id = currentInputTranscriptionId; + + // Add a special class to indicate it's a transcription + currentInputTranscriptionElement.classList.add("transcription"); + + messagesDiv.appendChild(currentInputTranscriptionElement); + } else { + // Update existing transcription bubble only if model hasn't started responding + // This prevents late partial transcriptions from overwriting complete ones + if (currentOutputTranscriptionId == null && currentMessageId == null) { + if (isFinished) { + // Final transcription contains the complete text, replace entirely + const cleanedText = cleanCJKSpaces(transcriptionText); + updateMessageBubble(currentInputTranscriptionElement, cleanedText, false); + } else { + // Partial transcription - append to existing text + const existingText = currentInputTranscriptionElement.querySelector(".bubble-text").textContent; + // Remove typing indicator if present + const cleanText = existingText.replace(/\.\.\.$/, ''); + // Clean spaces between CJK characters before updating + const accumulatedText = cleanCJKSpaces(cleanText + transcriptionText); + updateMessageBubble(currentInputTranscriptionElement, accumulatedText, true); + } + } + } + + // If transcription is finished, reset the state and mark as complete + if (isFinished) { + currentInputTranscriptionId = null; + currentInputTranscriptionElement = null; + inputTranscriptionFinished = true; // Prevent duplicate bubbles from late events + } + + scrollToBottom(); + } + } + + // Handle output transcription (model's spoken words) + if (adkEvent.outputTranscription && adkEvent.outputTranscription.text) { + const transcriptionText = adkEvent.outputTranscription.text; + const isFinished = adkEvent.outputTranscription.finished; + hasOutputTranscriptionInTurn = true; + + if (transcriptionText) { + // Finalize any active input transcription when server starts responding + if (currentInputTranscriptionId != null && currentOutputTranscriptionId == null) { + // This is the first output transcription - finalize input transcription + const textElement = currentInputTranscriptionElement.querySelector(".bubble-text"); + const typingIndicator = textElement.querySelector(".typing-indicator"); + if (typingIndicator) { + typingIndicator.remove(); + } + // Reset input transcription state so next user input creates new balloon + currentInputTranscriptionId = null; + currentInputTranscriptionElement = null; + inputTranscriptionFinished = true; // Prevent duplicate bubbles from late events + } + + if (currentOutputTranscriptionId == null) { + // Create new transcription bubble for agent + currentOutputTranscriptionId = Math.random().toString(36).substring(7); + currentOutputTranscriptionElement = createMessageBubble(transcriptionText, false, !isFinished); + currentOutputTranscriptionElement.id = currentOutputTranscriptionId; + + // Add a special class to indicate it's a transcription + currentOutputTranscriptionElement.classList.add("transcription"); + + messagesDiv.appendChild(currentOutputTranscriptionElement); + } else { + // Update existing transcription bubble + if (isFinished) { + // Final transcription contains the complete text, replace entirely + updateMessageBubble(currentOutputTranscriptionElement, transcriptionText, false); + } else { + // Partial transcription - append to existing text + const existingText = currentOutputTranscriptionElement.querySelector(".bubble-text").textContent; + // Remove typing indicator if present + const cleanText = existingText.replace(/\.\.\.$/, ''); + updateMessageBubble(currentOutputTranscriptionElement, cleanText + transcriptionText, true); + } + } + + // If transcription is finished, reset the state + if (isFinished) { + currentOutputTranscriptionId = null; + currentOutputTranscriptionElement = null; + } + + scrollToBottom(); + } + } + + // Handle content events (text or audio) + if (adkEvent.content && adkEvent.content.parts) { + const parts = adkEvent.content.parts; + + // Finalize any active input transcription when server starts responding with content + if (currentInputTranscriptionId != null && currentMessageId == null && currentOutputTranscriptionId == null) { + // This is the first content event - finalize input transcription + const textElement = currentInputTranscriptionElement.querySelector(".bubble-text"); + const typingIndicator = textElement.querySelector(".typing-indicator"); + if (typingIndicator) { + typingIndicator.remove(); + } + // Reset input transcription state so next user input creates new balloon + currentInputTranscriptionId = null; + currentInputTranscriptionElement = null; + inputTranscriptionFinished = true; // Prevent duplicate bubbles from late events + } + + for (const part of parts) { + // Handle inline data (audio) + if (part.inlineData) { + const mimeType = part.inlineData.mimeType; + const data = part.inlineData.data; + + if (mimeType && mimeType.startsWith("audio/pcm") && audioPlayerNode) { + audioPlayerNode.port.postMessage(base64ToArray(data)); + } + } + + // Handle text + if (part.text) { + // Skip thinking/reasoning text from chat bubbles (shown in event console) + if (part.thought) { + continue; + } + + // Skip final aggregated content when output transcription already + // delivered the response (prevents duplicate thinking text replay) + if (!adkEvent.partial && hasOutputTranscriptionInTurn) { + continue; + } + + // Add a new message bubble for a new turn + if (currentMessageId == null) { + currentMessageId = Math.random().toString(36).substring(7); + currentBubbleElement = createMessageBubble(part.text, false, true); + currentBubbleElement.id = currentMessageId; + messagesDiv.appendChild(currentBubbleElement); + } else { + // Update the existing message bubble with accumulated text + const existingText = currentBubbleElement.querySelector(".bubble-text").textContent; + // Remove the "..." if present + const cleanText = existingText.replace(/\.\.\.$/, ''); + updateMessageBubble(currentBubbleElement, cleanText + part.text, true); + } + + // Scroll down to the bottom of the messagesDiv + scrollToBottom(); + } + } + } + }; + + // Handle connection close + websocket.onclose = function () { + console.log("WebSocket connection closed."); + updateConnectionStatus(false); + document.getElementById("sendButton").disabled = true; + addSystemMessage("Connection closed. Reconnecting in 5 seconds..."); + + // Log to console + addConsoleEntry('error', 'WebSocket Disconnected', { + status: 'Connection closed', + reconnecting: true, + reconnectDelay: '5 seconds' + }, '🔌', 'system'); + + setTimeout(function () { + console.log("Reconnecting..."); + + // Log reconnection attempt to console + addConsoleEntry('outgoing', 'Reconnecting to ADK server...', { + userId: userId, + sessionId: sessionId + }, '🔄', 'system'); + + connectWebsocket(); + }, 5000); + }; + + websocket.onerror = function (e) { + console.log("WebSocket error: ", e); + updateConnectionStatus(false); + + // Log to console + addConsoleEntry('error', 'WebSocket Error', { + error: e.type, + message: 'Connection error occurred' + }, '⚠️', 'system'); + }; +} +connectWebsocket(); + +// Add submit handler to the form +function addSubmitHandler() { + messageForm.onsubmit = function (e) { + e.preventDefault(); + const message = messageInput.value.trim(); + if (message) { + // Add user message bubble + const userBubble = createMessageBubble(message, true, false); + messagesDiv.appendChild(userBubble); + scrollToBottom(); + + // Clear input + messageInput.value = ""; + + // Send message to server + sendMessage(message); + console.log("[CLIENT TO AGENT] " + message); + } + return false; + }; +} + +// Send a message to the server as JSON +function sendMessage(message) { + if (websocket && websocket.readyState == WebSocket.OPEN) { + const jsonMessage = JSON.stringify({ + type: "text", + text: message + }); + websocket.send(jsonMessage); + + // Log to console panel + addConsoleEntry('outgoing', 'User Message: ' + message, null, '💬', 'user'); + } +} + +// Decode Base64 data to Array +// Handles both standard base64 and base64url encoding +function base64ToArray(base64) { + // Convert base64url to standard base64 + // Replace URL-safe characters: - with +, _ with / + let standardBase64 = base64.replace(/-/g, '+').replace(/_/g, '/'); + + // Add padding if needed + while (standardBase64.length % 4) { + standardBase64 += '='; + } + + const binaryString = window.atob(standardBase64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes.buffer; +} + +/** + * Camera handling + */ + +const cameraButton = document.getElementById("cameraButton"); +const cameraModal = document.getElementById("cameraModal"); +const cameraPreview = document.getElementById("cameraPreview"); +const closeCameraModal = document.getElementById("closeCameraModal"); +const cancelCamera = document.getElementById("cancelCamera"); +const captureImageBtn = document.getElementById("captureImage"); + +let cameraStream = null; + +// Open camera modal and start preview +async function openCameraPreview() { + try { + // Request access to the user's webcam + cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { + width: { ideal: 768 }, + height: { ideal: 768 }, + facingMode: 'user' + } + }); + + // Set the stream to the video element + cameraPreview.srcObject = cameraStream; + + // Show the modal + cameraModal.classList.add('show'); + + } catch (error) { + console.error('Error accessing camera:', error); + addSystemMessage(`Failed to access camera: ${error.message}`); + + // Log to console + addConsoleEntry('error', 'Camera access failed', { + error: error.message, + name: error.name + }, '⚠️', 'system'); + } +} + +// Close camera modal and stop preview +function closeCameraPreview() { + // Stop the camera stream + if (cameraStream) { + cameraStream.getTracks().forEach(track => track.stop()); + cameraStream = null; + } + + // Clear the video source + cameraPreview.srcObject = null; + + // Hide the modal + cameraModal.classList.remove('show'); +} + +// Capture image from the live preview +function captureImageFromPreview() { + if (!cameraStream) { + addSystemMessage('No camera stream available'); + return; + } + + try { + // Create canvas to capture the frame + const canvas = document.createElement('canvas'); + canvas.width = cameraPreview.videoWidth; + canvas.height = cameraPreview.videoHeight; + const context = canvas.getContext('2d'); + + // Draw current video frame to canvas + context.drawImage(cameraPreview, 0, 0, canvas.width, canvas.height); + + // Convert canvas to data URL for display + const imageDataUrl = canvas.toDataURL('image/jpeg', 0.85); + + // Display the captured image in the chat + const imageBubble = createImageBubble(imageDataUrl, true); + messagesDiv.appendChild(imageBubble); + scrollToBottom(); + + // Convert canvas to blob for sending to server + canvas.toBlob((blob) => { + // Convert blob to base64 for sending to server + const reader = new FileReader(); + reader.onloadend = () => { + const base64data = reader.result.split(',')[1]; // Remove data:image/jpeg;base64, prefix + sendImage(base64data); + }; + reader.readAsDataURL(blob); + + // Log to console + addConsoleEntry('outgoing', `Image captured: ${blob.size} bytes (JPEG)`, { + size: blob.size, + type: 'image/jpeg', + dimensions: `${canvas.width}x${canvas.height}` + }, '📷', 'user'); + }, 'image/jpeg', 0.85); + + // Close the camera modal + closeCameraPreview(); + + } catch (error) { + console.error('Error capturing image:', error); + addSystemMessage(`Failed to capture image: ${error.message}`); + + // Log to console + addConsoleEntry('error', 'Image capture failed', { + error: error.message, + name: error.name + }, '⚠️', 'system'); + } +} + +// Send image to server +function sendImage(base64Image) { + if (websocket && websocket.readyState === WebSocket.OPEN) { + const jsonMessage = JSON.stringify({ + type: "image", + data: base64Image, + mimeType: "image/jpeg" + }); + websocket.send(jsonMessage); + console.log("[CLIENT TO AGENT] Sent image"); + } +} + +// Event listeners +cameraButton.addEventListener("click", openCameraPreview); +closeCameraModal.addEventListener("click", closeCameraPreview); +cancelCamera.addEventListener("click", closeCameraPreview); +captureImageBtn.addEventListener("click", captureImageFromPreview); + +// Close modal when clicking outside of it +cameraModal.addEventListener("click", (event) => { + if (event.target === cameraModal) { + closeCameraPreview(); + } +}); + +/** + * Audio handling + */ + +let audioPlayerNode; +let audioPlayerContext; +let audioRecorderNode; +let audioRecorderContext; +let micStream; + +// Import the audio worklets +import { startAudioPlayerWorklet } from "./audio-player.js"; +import { startAudioRecorderWorklet } from "./audio-recorder.js"; + +// Start audio +function startAudio() { + // Start audio output + startAudioPlayerWorklet().then(([node, ctx]) => { + audioPlayerNode = node; + audioPlayerContext = ctx; + }); + // Start audio input + startAudioRecorderWorklet(audioRecorderHandler).then( + ([node, ctx, stream]) => { + audioRecorderNode = node; + audioRecorderContext = ctx; + micStream = stream; + } + ); +} + +// Start the audio only when the user clicked the button +// (due to the gesture requirement for the Web Audio API) +const startAudioButton = document.getElementById("startAudioButton"); +startAudioButton.addEventListener("click", () => { + startAudioButton.disabled = true; + startAudio(); + is_audio = true; + addSystemMessage("Audio mode enabled - you can now speak to the agent"); + + // Log to console + addConsoleEntry('outgoing', 'Audio Mode Enabled', { + status: 'Audio worklets started', + message: 'Microphone active - audio input will be sent to agent' + }, '🎤', 'system'); +}); + +// Audio recorder handler +function audioRecorderHandler(pcmData) { + if (websocket && websocket.readyState === WebSocket.OPEN && is_audio) { + // Send audio as binary WebSocket frame (more efficient than base64 JSON) + websocket.send(pcmData); + console.log("[CLIENT TO AGENT] Sent audio chunk: %s bytes", pcmData.byteLength); + + // Log to console panel (optional, can be noisy with frequent audio chunks) + // addConsoleEntry('outgoing', `Audio chunk: ${pcmData.byteLength} bytes`); + } +} diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/audio-player.js b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/audio-player.js new file mode 100644 index 00000000000..c2a142530c5 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/audio-player.js @@ -0,0 +1,24 @@ +/** + * Audio Player Worklet + */ + +export async function startAudioPlayerWorklet() { + // 1. Create an AudioContext + const audioContext = new AudioContext({ + sampleRate: 24000 + }); + + + // 2. Load your custom processor code + const workletURL = new URL('./pcm-player-processor.js', import.meta.url); + await audioContext.audioWorklet.addModule(workletURL); + + // 3. Create an AudioWorkletNode + const audioPlayerNode = new AudioWorkletNode(audioContext, 'pcm-player-processor'); + + // 4. Connect to the destination + audioPlayerNode.connect(audioContext.destination); + + // The audioPlayerNode.port is how we send messages (audio data) to the processor + return [audioPlayerNode, audioContext]; +} diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/audio-recorder.js b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/audio-recorder.js new file mode 100644 index 00000000000..82806e6dd59 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/audio-recorder.js @@ -0,0 +1,58 @@ +/** + * Audio Recorder Worklet + */ + +let micStream; + +export async function startAudioRecorderWorklet(audioRecorderHandler) { + // Create an AudioContext + const audioRecorderContext = new AudioContext({ sampleRate: 16000 }); + console.log("AudioContext sample rate:", audioRecorderContext.sampleRate); + + // Load the AudioWorklet module + const workletURL = new URL("./pcm-recorder-processor.js", import.meta.url); + await audioRecorderContext.audioWorklet.addModule(workletURL); + + // Request access to the microphone + micStream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1 }, + }); + const source = audioRecorderContext.createMediaStreamSource(micStream); + + // Create an AudioWorkletNode that uses the PCMProcessor + const audioRecorderNode = new AudioWorkletNode( + audioRecorderContext, + "pcm-recorder-processor" + ); + + // Connect the microphone source to the worklet. + source.connect(audioRecorderNode); + audioRecorderNode.port.onmessage = (event) => { + // Convert to 16-bit PCM + const pcmData = convertFloat32ToPCM(event.data); + + // Send the PCM data to the handler. + audioRecorderHandler(pcmData); + }; + return [audioRecorderNode, audioRecorderContext, micStream]; +} + +/** + * Stop the microphone. + */ +export function stopMicrophone(micStream) { + micStream.getTracks().forEach((track) => track.stop()); + console.log("stopMicrophone(): Microphone stopped."); +} + +// Convert Float32 samples to 16-bit PCM. +function convertFloat32ToPCM(inputData) { + // Create an Int16Array of the same length. + const pcm16 = new Int16Array(inputData.length); + for (let i = 0; i < inputData.length; i++) { + // Multiply by 0x7fff (32767) to scale the float value to 16-bit PCM range. + pcm16[i] = inputData[i] * 0x7fff; + } + // Return the underlying ArrayBuffer. + return pcm16.buffer; +} diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/pcm-player-processor.js b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/pcm-player-processor.js new file mode 100644 index 00000000000..7e162362df5 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/pcm-player-processor.js @@ -0,0 +1,75 @@ +/** + * An audio worklet processor that stores the PCM audio data sent from the main thread + * to a buffer and plays it. + */ +class PCMPlayerProcessor extends AudioWorkletProcessor { + constructor() { + super(); + + // Init buffer + this.bufferSize = 24000 * 180; // 24kHz x 180 seconds + this.buffer = new Float32Array(this.bufferSize); + this.writeIndex = 0; + this.readIndex = 0; + + // Handle incoming messages from main thread + this.port.onmessage = (event) => { + // Reset the buffer when 'endOfAudio' message received + if (event.data.command === 'endOfAudio') { + this.readIndex = this.writeIndex; // Clear the buffer + console.log("endOfAudio received, clearing the buffer."); + return; + } + + // Decode the base64 data to int16 array. + const int16Samples = new Int16Array(event.data); + + // Add the audio data to the buffer + this._enqueue(int16Samples); + }; + } + + // Push incoming Int16 data into our ring buffer. + _enqueue(int16Samples) { + for (let i = 0; i < int16Samples.length; i++) { + // Convert 16-bit integer to float in [-1, 1] + const floatVal = int16Samples[i] / 32768; + + // Store in ring buffer for left channel only (mono) + this.buffer[this.writeIndex] = floatVal; + this.writeIndex = (this.writeIndex + 1) % this.bufferSize; + + // Overflow handling (overwrite oldest samples) + if (this.writeIndex === this.readIndex) { + this.readIndex = (this.readIndex + 1) % this.bufferSize; + } + } + } + + // The system calls `process()` ~128 samples at a time (depending on the browser). + // We fill the output buffers from our ring buffer. + process(inputs, outputs, parameters) { + + // Write a frame to the output + const output = outputs[0]; + const framesPerBlock = output[0].length; + for (let frame = 0; frame < framesPerBlock; frame++) { + + // Write the sample(s) into the output buffer + output[0][frame] = this.buffer[this.readIndex]; // left channel + if (output.length > 1) { + output[1][frame] = this.buffer[this.readIndex]; // right channel + } + + // Move the read index forward unless underflowing + if (this.readIndex != this.writeIndex) { + this.readIndex = (this.readIndex + 1) % this.bufferSize; + } + } + + // Returning true tells the system to keep the processor alive + return true; + } +} + +registerProcessor('pcm-player-processor', PCMPlayerProcessor); diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/pcm-recorder-processor.js b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/pcm-recorder-processor.js new file mode 100644 index 00000000000..b1c54bd3137 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/app/static/js/pcm-recorder-processor.js @@ -0,0 +1,18 @@ +class PCMProcessor extends AudioWorkletProcessor { + constructor() { + super(); + } + + process(inputs, outputs, parameters) { + if (inputs.length > 0 && inputs[0].length > 0) { + // Use the first channel + const inputChannel = inputs[0][0]; + // Copy the buffer to avoid issues with recycled memory + const inputCopy = new Float32Array(inputChannel); + this.port.postMessage(inputCopy); + } + return true; + } +} + +registerProcessor("pcm-recorder-processor", PCMProcessor); diff --git a/contributing/samples/live/live_bidi_streaming_demo_app/requirements.txt b/contributing/samples/live/live_bidi_streaming_demo_app/requirements.txt new file mode 100644 index 00000000000..92b713e8e85 --- /dev/null +++ b/contributing/samples/live/live_bidi_streaming_demo_app/requirements.txt @@ -0,0 +1,4 @@ +fastapi +google-adk>=2.5.0 +python-dotenv +uvicorn[standard]