From 95b86e35356913ffa02b408194ce5f14461db6f8 Mon Sep 17 00:00:00 2001
From: JensLincke <jens.lincke@hpi.uni-potsdam.de>
Date: Fri, 26 Apr 2024 10:33:54 +0200
Subject: [PATCH] new openai text to speach tool

SQUASHED: AUTO-COMMIT-doc-journal-2024-04-25.md-index.md,AUTO-COMMIT-src-components-tools-openai-audio-speech.html,AUTO-COMMIT-src-components-tools-openai-audio-speech.js,
---
 doc/journal/2024-04-25.md/index.md            |  47 ++++++++
 src/components/tools/openai-audio-speech.html |  33 ++++++
 src/components/tools/openai-audio-speech.js   | 110 ++++++++++++++++++
 3 files changed, 190 insertions(+)
 create mode 100644 doc/journal/2024-04-25.md/index.md
 create mode 100644 src/components/tools/openai-audio-speech.html
 create mode 100644 src/components/tools/openai-audio-speech.js

diff --git a/doc/journal/2024-04-25.md/index.md b/doc/journal/2024-04-25.md/index.md
new file mode 100644
index 000000000..0696cc8b9
--- /dev/null
+++ b/doc/journal/2024-04-25.md/index.md
@@ -0,0 +1,47 @@
+## 2024-04-25 OpenAI text to speech
+*Author: @JensLincke*
+
+This should either go into a tool ore context menu
+
+
+```javascript
+import OpenAI from "demos/openai/openai.js"
+
+let apiKey = await OpenAI.ensureSubscriptionKey()
+
+let prompt = {
+    "model": "tts-1",
+    "input": that.value,
+    "voice": "alloy"
+  }
+
+
+const url = "https://api.openai.com/v1/audio/speech";
+
+const requestOptions = {
+  method: "POST",
+  headers: {
+    "Content-Type": "application/json",
+    "Authorization": `Bearer ${apiKey}`
+  },
+  body: JSON.stringify(prompt)
+};
+
+const ctx = new AudioContext();
+
+let result = await fetch(url, requestOptions)
+
+
+let audio = await result.arrayBuffer().then(arrayBuffer => ctx.decodeAudioData(arrayBuffer))
+	
+
+function playback() {
+  const playSound = ctx.createBufferSource();
+  playSound.buffer = audio;
+  playSound.connect(ctx.destination);
+  playSound.start(ctx.currentTime);
+}
+
+playback()
+```
+
diff --git a/src/components/tools/openai-audio-speech.html b/src/components/tools/openai-audio-speech.html
new file mode 100644
index 000000000..ba6d29c9a
--- /dev/null
+++ b/src/components/tools/openai-audio-speech.html
@@ -0,0 +1,33 @@
+<template id="openai-audio-speech">
+  <style data-src="/src/external/font-awesome/css/font-awesome.css"></style>
+  <style data-src="/templates/livelystyle.css"></style>
+  <style>
+    :host {
+      
+      
+    }
+    #editor {
+      max-height: calc(100%);
+      flex: 1; 
+    }
+
+    #pane {
+     width: 100%;
+      height: 100%;
+      display: flex;
+      flex-direction: column;
+    } 
+    
+    #player {
+      float: right
+    }
+    
+  </style>
+  <div id="pane">
+    <div>
+      <button id="generate">generate</button>
+      <audio id="player" controls="true"></audio>  
+    </div>
+    <lively-code-mirror id="editor"></lively-code-mirror>
+  </div>
+</template>
diff --git a/src/components/tools/openai-audio-speech.js b/src/components/tools/openai-audio-speech.js
new file mode 100644
index 000000000..07112a7f8
--- /dev/null
+++ b/src/components/tools/openai-audio-speech.js
@@ -0,0 +1,110 @@
+import OpenAI from "demos/openai/openai.js"
+
+import Morph from 'src/components/widgets/lively-morph.js';
+
+export default class OpenaiAudioSpeech extends Morph {
+  async initialize() {
+    this.windowTitle = "OpenaiAudioSpeech";
+    this.registerButtons()
+    lively.html.registerKeys(this); // automatically installs handler for some methods
+
+    this.editor.value = this.getAttribute("value")
+
+    this.editor.addEventListener("editor-loaded", () => {
+      this.editor.setOption('lineWrapping', true)
+
+      this.editor.editor.on("change", cm => {
+        this.setAttribute("value", this.editor.value)
+      })
+
+    })
+
+  }
+
+  get editor() {
+    return this.get("#editor")
+  }
+
+  get player() {
+    return this.get("#player")
+  }
+
+
+  get text() {
+    return this.editor.value
+  }
+
+
+  setupMediaSource() {
+    const mediaSource = new MediaSource();
+    const audio = this.player;
+    audio.controls = true;
+    audio.src = URL.createObjectURL(mediaSource);
+
+    mediaSource.addEventListener('sourceopen', () => {
+      console.log("sourceopen")
+      const sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg'); // Use the appropriate MIME type
+
+      // Fetch the data and append it to the source buffer
+      this.fetchDataAndAppend(mediaSource, sourceBuffer);
+    });
+
+    audio.play()
+
+    return audio;
+  }
+
+  // This function fetches audio data using POST and appends chunks to the source buffer.
+  async fetchDataAndAppend(mediaSource, sourceBuffer) {
+
+    let apiKey = await OpenAI.ensureSubscriptionKey()
+
+    let prompt = {
+      "model": "tts-1",
+      "input": this.text,
+      "voice": "alloy"
+    }
+
+
+    const url = "https://api.openai.com/v1/audio/speech";
+
+    const requestOptions = {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`
+      },
+      body: JSON.stringify(prompt)
+    };
+
+
+    const response = await fetch(url, requestOptions)
+    const reader = response.body.getReader();
+
+    // Function to handle reading each chunk
+    function process({ done, value }) {
+      if (done) {
+        mediaSource.endOfStream(); // Properly call endOfStream on the MediaSource instance
+        return;
+      }
+      if (sourceBuffer.updating) {
+        // If buffer is still updating, wait before appending more data
+        setTimeout(() => reader.read().then(process), 100);
+      } else {
+        sourceBuffer.appendBuffer(value);
+        reader.read().then(process);
+      }
+    }
+
+    // Start processing the stream
+    reader.read().then(process).catch(error => {
+      console.error('Error fetching or processing data:', error);
+      mediaSource.endOfStream('network'); // Signal an error in fetching stream
+    });
+  }
+
+  async onGenerate() {
+    // Call this function to start the process.
+    this.setupMediaSource();
+  }
+}