feat(genai): live samples part2

genai/live/live-audio-with-txt.js

@@ -0,0 +1,122 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START googlegenaisdk_live_audio_with_txt]
+
+'use strict';
+
+const {GoogleGenAI, Modality} = require('@google/genai');
+const fs = require('fs');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+async function generateLiveConversation(
+  projectId = GOOGLE_CLOUD_PROJECT,
+  location = GOOGLE_CLOUD_LOCATION
+) {
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: projectId,
+    location: location,
+  });
+
+  const voiceName = 'Aoede';
+  const modelId = 'gemini-2.0-flash-live-preview-04-09';
+  const config = {
+    responseModalities: [Modality.AUDIO],
+    speechConfig: {
+      voiceConfig: {
+        prebuiltVoiceConfig: {
+          voiceName: voiceName,
+        },
+      },
+    },
+  };
+
+  const responseQueue = [];
+
+  async function waitMessage() {
+    while (responseQueue.length === 0) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    return responseQueue.shift();
+  }
+
+  async function handleTurn() {
+    const audioChunks = [];
+    let done = false;
+
+    while (!done) {
+      const message = await waitMessage();
+
+      const serverContent = message.serverContent;
+      if (
+        serverContent &&
+        serverContent.modelTurn &&
+        serverContent.modelTurn.parts
+      ) {
+        for (const part of serverContent.modelTurn.parts) {
+          if (part && part.inlineData && part.inlineData.data) {
+            audioChunks.push(Buffer.from(part.inlineData.data));
+          }
+        }
+      }
+
+      if (serverContent && serverContent.turnComplete) {
+        done = true;
+      }
+    }
+
+    return audioChunks;
+  }
+
+  const session = await client.live.connect({
+    model: modelId,
+    config: config,
+    callbacks: {
+      onmessage: msg => responseQueue.push(msg),
+      onerror: e => console.error('Error:', e.message),
+    },
+  });
+
+  const textInput = 'Hello? Gemini are you there?';
+  console.log('> ', textInput, '\n');
+
+  await session.sendClientContent({
+    turns: [{role: 'user', parts: [{text: textInput}]}],
+  });
+
+  const audioChunks = await handleTurn();
+
+  session.close();
+
+  if (audioChunks.length > 0) {
+    const audioBuffer = Buffer.concat(audioChunks);
+    fs.writeFileSync('response.raw', audioBuffer);
+    console.log('Received audio answer (saved to response.raw)');
+  }
+
+  // Example output:
+  //> Hello? Gemini, are you there?
+  // Received audio answer (saved to response.raw)
+
+  return audioChunks;
+}
+
+// [END googlegenaisdk_live_audio_with_txt]
+
+module.exports = {
+  generateLiveConversation,
+};

genai/live/live-ground-ragengine-with-txt.js

@@ -0,0 +1,124 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START googlegenaisdk_live_ground_ragengine_with_txt]
+
+'use strict';
+
+const {GoogleGenAI, Modality} = require('@google/genai');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+// (DEVELOPER) put here your memory corpus
+const MEMORY_CORPUS =
+  'projects/cloud-ai-devrel-softserve/locations/us-central1/ragCorpora/2305843009213693952';
+
+async function generateLiveRagTextResponse(
+  memoryCorpus = MEMORY_CORPUS,
+  projectId = GOOGLE_CLOUD_PROJECT,
+  location = GOOGLE_CLOUD_LOCATION
+) {
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: projectId,
+    location: location,
+  });
+
+  const modelId = 'gemini-2.0-flash-live-preview-04-09';
+
+  // RAG store config
+  const ragStore = {
+    ragResources: [
+      {
+        ragCorpus: memoryCorpus, // Use memory corpus if you want to store context
+      },
+    ],
+    storeContext: true, // sink context into your memory corpus
+  };
+
+  const config = {
+    responseModalities: [Modality.TEXT],
+    tools: [
+      {
+        retrieval: {
+          vertexRagStore: ragStore,
+        },
+      },
+    ],
+  };
+
+  const responseQueue = [];
+
+  async function waitMessage() {
+    while (responseQueue.length === 0) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    return responseQueue.shift();
+  }
+
+  async function handleTurn() {
+    const turns = [];
+    let done = false;
+    while (!done) {
+      const message = await waitMessage();
+      turns.push(message);
+      if (message.serverContent && message.serverContent.turnComplete) {
+        done = true;
+      }
+    }
+    return turns;
+  }
+
+  const session = await client.live.connect({
+    model: modelId,
+    config: config,
+    callbacks: {
+      onmessage: msg => responseQueue.push(msg),
+      onerror: e => console.error('Error:', e.message),
+    },
+  });
+
+  const textInput = 'What are newest gemini models?';
+  console.log('> ', textInput, '\n');
+
+  await session.sendClientContent({
+    turns: [{role: 'user', parts: [{text: textInput}]}],
+  });
+
+  const turns = await handleTurn();
+  const response = [];
+
+  for (const turn of turns) {
+    if (turn.text) {
+      response.push(turn.text);
+    }
+  }
+
+  console.log(response.join(''));
+
+  // Example output:
+  // > What are newest gemini models?
+  //  In December 2023, Google launched Gemini, their "most capable and general model". It's multimodal, meaning it understands and combines different types of information like text, code, audio, images, and video.
+
+  session.close();
+
+  return response;
+}
+
+// [END googlegenaisdk_live_ground_ragengine_with_txt]
+
+module.exports = {
+  generateLiveRagTextResponse,
+};

genai/live/live-structured-ouput-with-txt.js

@@ -0,0 +1,93 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START googlegenaisdk_live_structured_output_with_txt]
+
+'use strict';
+const {OpenAI} = require('openai');
+const {GoogleAuth} = require('google-auth-library');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION =
+  process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';
+
+const CalendarEventSchema = {
+  type: 'object',
+  properties: {
+    name: {type: 'string'},
+    date: {type: 'string'},
+    participants: {
+      type: 'array',
+      items: {type: 'string'},
+    },
+  },
+  required: ['name', 'date', 'participants'],
+};
+
+async function generateStructuredTextResponse(
+  projectId = GOOGLE_CLOUD_PROJECT,
+  location = GOOGLE_CLOUD_LOCATION
+) {
+  const auth = new GoogleAuth({
+    scopes: ['https://www.googleapis.com/auth/cloud-platform'],
+  });
+  const client = await auth.getClient();
+  const tokenResponse = await client.getAccessToken();
+
+  const token = tokenResponse.token;
+
+  const ENDPOINT_ID = 'openapi';
+  const baseURL = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/endpoints/${ENDPOINT_ID}`;
+
+  const openAI = new OpenAI({
+    apiKey: token,
+    baseURL: baseURL,
+  });
+
+  const completion = await openAI.chat.completions.create({
+    model: 'google/gemini-2.0-flash-001',
+    messages: [
+      {role: 'system', content: 'Extract the event information.'},
+      {
+        role: 'user',
+        content: 'Alice and Bob are going to a science fair on Friday.',
+      },
+    ],
+    response_format: {
+      type: 'json_schema',
+      json_schema: {
+        name: 'CalendarEvent',
+        schema: CalendarEventSchema,
+      },
+    },
+  });
+
+  const response = completion.choices[0].message.content;
+  console.log(response);
+
+  // Example expected output:
+  // {
+  //   name: 'science fair',
+  //   date: 'Friday',
+  //   participants: ['Alice', 'Bob']
+  // }
+
+  return response;
+}
+
+// [END googlegenaisdk_live_structured_output_with_txt]
+
+module.exports = {
+  generateStructuredTextResponse,
+};