Google Gemini Audio Webinar.ipynb

{"cells":[{"attachments":{},"cell_type":"markdown","metadata":{"id":"0etRtS83RcWS"},"source":"# Gemini API: Audio Quickstart"},{"attachments":{},"cell_type":"markdown","metadata":{"id":"r1IzNLho-NqV"},"source":"This notebook provides an example of how to prompt Gemini 1.5 Pro using an audio file. "},{"cell_type":"code","execution_count":159,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T09:03:53.160734Z","iopub.status.busy":"2024-04-26T09:03:53.160355Z","iopub.status.idle":"2024-04-26T09:03:56.233395Z","shell.execute_reply":"2024-04-26T09:03:56.232731Z","shell.execute_reply.started":"2024-04-26T09:03:53.160719Z"},"id":"Y6eH_Aq_NyNi","language":"python","trusted":true},"outputs":[],"source":"!pip install -q -U google-generativeai langchain langchain-google-genai langchain-openai singlestoredb --quiet"},{"cell_type":"code","execution_count":110,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T08:55:38.215642Z","iopub.status.busy":"2024-04-26T08:55:38.215174Z","iopub.status.idle":"2024-04-26T08:55:38.217869Z","shell.execute_reply":"2024-04-26T08:55:38.217441Z","shell.execute_reply.started":"2024-04-26T08:55:38.215628Z"},"id":"LSe1pMEpR2L2","language":"python","trusted":true},"outputs":[],"source":"import google.generativeai as genai"},{"attachments":{},"cell_type":"markdown","metadata":{"id":"TXiv-NeZR5WA"},"source":"## Configure your API key\n\nThis API key will be from aistudio.google.com"},{"cell_type":"code","execution_count":111,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T08:55:41.177966Z","iopub.status.busy":"2024-04-26T08:55:41.177496Z","iopub.status.idle":"2024-04-26T08:55:41.180370Z","shell.execute_reply":"2024-04-26T08:55:41.179981Z","shell.execute_reply.started":"2024-04-26T08:55:41.177950Z"},"id":"dm-iaNMGPdid","language":"python","trusted":true},"outputs":[],"source":"import os\n\nos.environ['GOOGLE_API_KEY']='ap'\n\ngenai.configure(api_key=os.environ['GOOGLE_API_KEY'])"},{"attachments":{},"cell_type":"markdown","metadata":{"id":"2YoxMrCdR7hf"},"source":"## Upload an audio file with the File API\n\nTo use an audio file in your prompt, you must first upload it using the [File API](https://github.com/google-gemini/cookbook/blob/main/quickstarts/File_API.ipynb).\n"},{"cell_type":"code","execution_count":112,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T08:55:42.502583Z","iopub.status.busy":"2024-04-26T08:55:42.502121Z","iopub.status.idle":"2024-04-26T08:55:42.504725Z","shell.execute_reply":"2024-04-26T08:55:42.504342Z","shell.execute_reply.started":"2024-04-26T08:55:42.502569Z"},"id":"OHvNLws4RRjx","language":"python","trusted":true},"outputs":[],"source":"URL = \"https://ia803402.us.archive.org/14/items/lp_mozart-divertimento17-k-334-horn-quintet-k_wolfgang-amadeus-mozart-members-of-the-ber/disc1/01.03.%20Divertmento%20In%20D%20Major%2C%20K.%20334%20Menuetto.mp3\""},{"cell_type":"code","execution_count":114,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T08:55:43.968356Z","iopub.status.busy":"2024-04-26T08:55:43.967887Z","iopub.status.idle":"2024-04-26T08:55:48.422874Z","shell.execute_reply":"2024-04-26T08:55:48.422250Z","shell.execute_reply.started":"2024-04-26T08:55:43.968341Z"},"id":"Cxq31LDwSFH6","language":"python","trusted":true},"outputs":[],"source":"!wget -q $URL -O sample.mp3"},{"cell_type":"code","execution_count":115,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T08:55:49.566063Z","iopub.status.busy":"2024-04-26T08:55:49.565548Z","iopub.status.idle":"2024-04-26T08:55:50.743497Z","shell.execute_reply":"2024-04-26T08:55:50.743033Z","shell.execute_reply.started":"2024-04-26T08:55:49.566039Z"},"id":"MAObE0BpaAwG","language":"python","trusted":true},"outputs":[],"source":"your_file = genai.upload_file(path='sample.mp3')"},{"attachments":{},"cell_type":"markdown","metadata":{"id":"m01XDoo4UQvN"},"source":"## Use the file in your prompt"},{"cell_type":"code","execution_count":122,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":73},"execution":{"iopub.execute_input":"2024-04-26T08:56:33.801544Z","iopub.status.busy":"2024-04-26T08:56:33.801267Z","iopub.status.idle":"2024-04-26T08:56:35.979751Z","shell.execute_reply":"2024-04-26T08:56:35.979079Z","shell.execute_reply.started":"2024-04-26T08:56:33.801529Z"},"id":"YmISEsqpafRb","language":"python","outputId":"b5873da2-7f3c-4fc7-9ec5-f5c74aad212f","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"The audio file contains a series of classical music pieces featuring piano, strings, and woodwinds. \n\n"}],"source":"prompt = \"Listen carefully to the following audio file. Provide a one sentence summary.\"\nmodel = genai.GenerativeModel('models/gemini-1.5-pro-latest')\nresponse = model.generate_content([prompt, your_file])\nprint(response.text)"},{"attachments":{},"cell_type":"markdown","metadata":{"execution":{"iopub.execute_input":"2024-04-26T08:44:11.362703Z","iopub.status.busy":"2024-04-26T08:44:11.362215Z","iopub.status.idle":"2024-04-26T08:44:11.364950Z","shell.execute_reply":"2024-04-26T08:44:11.364515Z","shell.execute_reply.started":"2024-04-26T08:44:11.362679Z"},"language":"python"},"source":"## RAG over audio files using SingleStoreDB\n\nNow we will embed the text descriptions of the audio file(s). This allows us to search and retrieve relevant files for RAG later."},{"cell_type":"code","execution_count":172,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T09:06:00.761738Z","iopub.status.busy":"2024-04-26T09:06:00.761467Z","iopub.status.idle":"2024-04-26T09:06:00.764687Z","shell.execute_reply":"2024-04-26T09:06:00.764281Z","shell.execute_reply.started":"2024-04-26T09:06:00.761721Z"},"language":"python","trusted":true},"outputs":[],"source":"from langchain.vectorstores import SingleStoreDB\nimport os\n\nfrom langchain_google_genai import GoogleGenerativeAIEmbeddings\n\nembeddings = GoogleGenerativeAIEmbeddings(model=\"models/embedding-001\")\n\nos.environ[\"SINGLESTOREDB_URL\"] = f'{connection_user}:{connection_password}@{connection_host}:{connection_port}/{connection_default_database}'"},{"cell_type":"code","execution_count":244,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T09:18:44.313877Z","iopub.status.busy":"2024-04-26T09:18:44.313602Z","iopub.status.idle":"2024-04-26T09:18:45.005645Z","shell.execute_reply":"2024-04-26T09:18:45.005187Z","shell.execute_reply.started":"2024-04-26T09:18:44.313862Z"},"language":"python","trusted":true},"outputs":[],"source":"vectorstore=SingleStoreDB(table_name=\"audio1\", embedding=embeddings)"},{"cell_type":"code","execution_count":248,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T09:18:46.410577Z","iopub.status.busy":"2024-04-26T09:18:46.410062Z","iopub.status.idle":"2024-04-26T09:18:47.338408Z","shell.execute_reply":"2024-04-26T09:18:47.338025Z","shell.execute_reply.started":"2024-04-26T09:18:46.410562Z"},"language":"python","trusted":true},"outputs":[{"data":{"text/plain":"[]"},"execution_count":248,"metadata":{},"output_type":"execute_result"}],"source":"from langchain_core.documents import Document\n\nmozart_doc = Document(page_content=response.text, metadata={'path': 'sample.mp3'})\n\nvectorstore.add_documents([mozart_doc])   \n\nvectorstore.add_texts(['foo', 'bar'])"},{"cell_type":"code","execution_count":259,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T09:19:39.907371Z","iopub.status.busy":"2024-04-26T09:19:39.906838Z","iopub.status.idle":"2024-04-26T09:19:40.246762Z","shell.execute_reply":"2024-04-26T09:19:40.246307Z","shell.execute_reply.started":"2024-04-26T09:19:39.907355Z"},"language":"python","trusted":true},"outputs":[],"source":"query = \"beethoven\"\ndocs = vectorstore.similarity_search(query)  # Find documents that correspond to the query"},{"cell_type":"code","execution_count":260,"metadata":{"execution":{"iopub.execute_input":"2024-04-26T09:19:41.271503Z","iopub.status.busy":"2024-04-26T09:19:41.271084Z","iopub.status.idle":"2024-04-26T09:19:41.274037Z","shell.execute_reply":"2024-04-26T09:19:41.273647Z","shell.execute_reply.started":"2024-04-26T09:19:41.271489Z"},"language":"python","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"The audio file contains a series of classical music pieces featuring piano, strings, and woodwinds. \n\n"}],"source":"print(docs[-1].page_content)"}],"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.6"},"singlestore_cell_default_language":"python","singlestore_connection":{"connectionID":"6efce35a-2db5-4ae3-bc0b-c3d5810a45f4","defaultDatabase":"langchain_multimodal"}},"nbformat":4,"nbformat_minor":4}