refactor: update real-time input message structure to differentiate between text, audio, and video types.

Author: thorwebdev

gemini-live-ephemeral-tokens-websocket/frontend/geminilive.js

@@ -368,18 +368,12 @@ class GeminiLiveAPI {
   }
 
   sendTextMessage(text) {
-    const textMessage = {
-      clientContent: {
-        turns: [
-          {
-            role: "user",
-            parts: [{ text: text }],
-          },
-        ],
-        turnComplete: true,
+    const message = {
+      realtimeInput: {
+        text: text,
       },
     };
-    this.sendMessage(textMessage);
+    this.sendMessage(message);
   }
 
   sendToolResponse(toolCallId, response) {
@@ -394,16 +388,15 @@ class GeminiLiveAPI {
   }
 
   sendRealtimeInputMessage(data, mimeType) {
-    const message = {
-      realtimeInput: {
-        mediaChunks: [
-          {
-            mimeType: mimeType,
-            data: data,
-          },
-        ],
-      },
-    };
+    const blob = { mimeType, data };
+    const message = { realtimeInput: {} };
+
+    if (mimeType.startsWith("audio/")) {
+      message.realtimeInput.audio = blob;
+    } else if (mimeType.startsWith("image/") || mimeType.startsWith("video/")) {
+      message.realtimeInput.video = blob;
+    }
+
     this.sendMessage(message);
     this.addToBytesSent(data);
   }

gemini-live-genai-python-sdk/gemini_live.py

@@ -64,7 +64,7 @@ async def send_video():
                         chunk = await video_input_queue.get()
                         logger.info(f"Sending video frame to Gemini: {len(chunk)} bytes")
                         await session.send_realtime_input(
-                            media=types.Blob(data=chunk, mime_type="image/jpeg")
+                            video=types.Blob(data=chunk, mime_type="image/jpeg")
                         )
                 except asyncio.CancelledError:
                     pass