Expose functions from AudioStreamPlaybackMicrophone to access the microphone AudioDriver::input_buffer independently of the rest of the Audio system

doc/classes/AudioStreamPlaybackMicrophone.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="AudioStreamPlaybackMicrophone" inherits="AudioStreamPlaybackResampled" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
+	<brief_description>
+		Playback instance for [AudioStreamMicrophone].
+	</brief_description>
+	<description>
+		Playback instance for [AudioStreamMicrophone]. When this class is instantiated on its own it can be used to control and extract the microphone audio data directly.
+	</description>
+	<tutorials>
+	</tutorials>
+	<methods>
+		<method name="get_microphone_buffer">
+			<return type="PackedVector2Array" />
+			<param index="0" name="frames" type="int" />
+			<description>
+				Gets the next [param frames] audio samples from the AudioDevice ring buffer.
+				Returns a [PackedVector2Array] containing exactly [param frames] audio samples if available, or an empty [PackedVector2Array] if insufficient data was available.
+				The samples are signed floating-point PCM between [code]-1[/code] and [code]1[/code]. You will have to scale them if you want to use them as 8 or 16-bit integer samples. ([code]v = 0x7fff * samples[0].x[/code])
+			</description>
+		</method>
+		<method name="is_microphone_playing" qualifiers="const">
+			<return type="bool" />
+			<description>
+				Check if this object has instructed the AudioDevice to record.
+			</description>
+		</method>
+		<method name="start_microphone">
+			<return type="void" />
+			<description>
+				Instruct the AudioDevice to begin recording if it has not already doing so.
+			</description>
+		</method>
+		<method name="stop_microphone">
+			<return type="void" />
+			<description>
+				Instruct the AudioDevice to stop recording if no other [AudioStreamPlaybackMicrophone] instances are active.
+			</description>
+		</method>
+	</methods>
+</class>

drivers/coreaudio/audio_driver_coreaudio.h

@@ -118,7 +118,7 @@ class AudioDriverCoreAudio : public AudioDriver {
 	virtual Error input_start() override;
 	virtual Error input_stop() override;
 
-	bool try_lock();
+	virtual bool try_lock() override;
 	void stop();
 
 	AudioDriverCoreAudio();

drivers/pulseaudio/audio_driver_pulseaudio.cpp

@@ -537,6 +537,7 @@ void AudioDriverPulseAudio::thread_func(void *p_udata) {
 					ERR_PRINT("pa_stream_peek error");
 				} else {
 					int16_t *srcptr = (int16_t *)ptr;
+					ad->lock();
 					for (size_t i = bytes >> 1; i > 0; i--) {
 						int32_t sample = int32_t(*srcptr++) << 16;
 						ad->input_buffer_write(sample);
@@ -546,6 +547,7 @@ void AudioDriverPulseAudio::thread_func(void *p_udata) {
 							ad->input_buffer_write(sample);
 						}
 					}
+					ad->unlock();
 
 					read_bytes += bytes;
 					ret = pa_stream_drop(ad->pa_rec_str);

drivers/wasapi/audio_driver_wasapi.cpp

@@ -848,6 +848,7 @@ void AudioDriverWASAPI::thread_func(void *p_udata) {
 					ERR_BREAK(hr != S_OK);
 
 					// fixme: Only works for floating point atm
+					ad->lock();
 					for (UINT32 j = 0; j < num_frames_available; j++) {
 						int32_t l, r;
 
@@ -868,6 +869,7 @@ void AudioDriverWASAPI::thread_func(void *p_udata) {
 						ad->input_buffer_write(l);
 						ad->input_buffer_write(r);
 					}
+					ad->unlock();
 
 					read_frames += num_frames_available;
 

platform/android/audio_driver_opensl.cpp

@@ -184,11 +184,13 @@ void AudioDriverOpenSL::start() {
 }
 
 void AudioDriverOpenSL::_record_buffer_callback(SLAndroidSimpleBufferQueueItf queueItf) {
+	lock();
 	for (int i = 0; i < rec_buffer.size(); i++) {
 		int32_t sample = rec_buffer[i] << 16;
 		input_buffer_write(sample);
 		input_buffer_write(sample); // call twice to convert to Stereo
 	}
+	unlock();
 
 	SLresult res = (*recordBufferQueueItf)->Enqueue(recordBufferQueueItf, rec_buffer.ptrw(), rec_buffer.size() * sizeof(int16_t));
 	ERR_FAIL_COND(res != SL_RESULT_SUCCESS);
@@ -272,29 +274,30 @@ Error AudioDriverOpenSL::input_start() {
 		return ERR_ALREADY_IN_USE;
 	}
 
-	if (OS::get_singleton()->request_permission("RECORD_AUDIO")) {
-		return init_input_device();
+	if (!OS::get_singleton()->request_permission("RECORD_AUDIO")) {
+		WARN_PRINT("Unable to start audio capture - No RECORD_AUDIO permission -- yet");
+		return ERR_UNAUTHORIZED;
 	}
-
-	WARN_PRINT("Unable to start audio capture - No RECORD_AUDIO permission");
-	return ERR_UNAUTHORIZED;
+	return init_input_device();
 }
 
 Error AudioDriverOpenSL::input_stop() {
-	if (!recordItf || !recordBufferQueueItf) {
-		return ERR_CANT_OPEN;
+	//if (!recordItf || !recordBufferQueueItf) {
+	//	return ERR_CANT_OPEN;
+	//}
+	if (recordItf) {
+		(*recordItf)->SetRecordState(recordItf, SL_RECORDSTATE_STOPPED);
+		recordItf = nullptr;
 	}
 
-	SLuint32 state;
-	SLresult res = (*recordItf)->GetRecordState(recordItf, &state);
-	ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN);
-
-	if (state != SL_RECORDSTATE_STOPPED) {
-		res = (*recordItf)->SetRecordState(recordItf, SL_RECORDSTATE_STOPPED);
-		ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN);
+	if (recordBufferQueueItf) {
+		(*recordBufferQueueItf)->Clear(recordBufferQueueItf);
+		recordBufferQueueItf = nullptr;
+	}
 
-		res = (*recordBufferQueueItf)->Clear(recordBufferQueueItf);
-		ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN);
+	if (recorder) {
+		(*recorder)->Destroy(recorder);
+		recorder = nullptr;
 	}
 
 	return OK;
@@ -314,6 +317,13 @@ void AudioDriverOpenSL::lock() {
 	}
 }
 
+bool AudioDriverOpenSL::try_lock() {
+	if (active) {
+		return mutex.try_lock();
+	}
+	return true;
+}
+
 void AudioDriverOpenSL::unlock() {
 	if (active) {
 		mutex.unlock();

platform/android/audio_driver_opensl.h

@@ -97,6 +97,7 @@ class AudioDriverOpenSL : public AudioDriver {
 	virtual SpeakerMode get_speaker_mode() const override;
 
 	virtual void lock() override;
+	virtual bool try_lock() override;
 	virtual void unlock() override;
 	virtual void finish() override;
 

platform/android/java_godot_lib_jni.cpp

@@ -566,9 +566,12 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_filePickerCallback(JN
 
 JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_requestPermissionResult(JNIEnv *env, jclass clazz, jstring p_permission, jboolean p_result) {
 	String permission = jstring_to_string(p_permission, env);
-	if (permission == "android.permission.RECORD_AUDIO" && p_result) {
-		AudioDriver::get_singleton()->input_start();
-	}
+
+	// this code block is not helpful due to the lack of error handling and the
+	// information not getting back to the AudioStreamPlaybackMicrophone
+	//if (permission == "android.permission.RECORD_AUDIO" && p_result) {
+	//	AudioDriver::get_singleton()->input_start();
+	//}
 
 	if (os_android->get_main_loop()) {
 		os_android->get_main_loop()->emit_signal(SNAME("on_request_permissions_result"), permission, p_result == JNI_TRUE);

platform/web/audio_driver_web.cpp

@@ -105,6 +105,8 @@ void AudioDriverWeb::_audio_driver_capture(int p_from, int p_samples) {
 	if (to_read == 0) {
 		to_read = max_samples;
 	}
+
+	lock();
 	// High part
 	if (read_pos + to_read > max_samples) {
 		const int samples_high = max_samples - read_pos;
@@ -118,6 +120,7 @@ void AudioDriverWeb::_audio_driver_capture(int p_from, int p_samples) {
 	for (int i = read_pos; i < read_pos + to_read; i++) {
 		input_buffer_write(int32_t(input_rb[i] * 32768.f) * (1U << 16));
 	}
+	unlock();
 }
 
 Error AudioDriverWeb::init() {

servers/audio/audio_stream.cpp

@@ -390,7 +390,7 @@ bool AudioStreamMicrophone::is_monophonic() const {
 int AudioStreamPlaybackMicrophone::_mix_internal(AudioFrame *p_buffer, int p_frames) {
 	AudioDriver::get_singleton()->lock();
 
-	Vector<int32_t> buf = AudioDriver::get_singleton()->get_input_buffer();
+	Vector<int32_t> &buf = AudioDriver::get_singleton()->get_input_buffer();
 	unsigned int input_size = AudioDriver::get_singleton()->get_input_size();
 	int mix_rate = AudioDriver::get_singleton()->get_input_mix_rate();
 	unsigned int playback_delay = MIN(((50 * mix_rate) / 1000) * 2, buf.size() >> 1);
@@ -438,6 +438,59 @@ int AudioStreamPlaybackMicrophone::_mix_internal(AudioFrame *p_buffer, int p_fra
 	return mixed_frames;
 }
 
+PackedVector2Array AudioStreamPlaybackMicrophone::get_microphone_buffer(int p_frames) {
+	PackedVector2Array ret;
+	if (!microphone.is_null()) {
+		WARN_PRINT("You cannot call get_microphone_buffer() on AudioStreamPlaybackMicrophone if it is part of a stream.");
+		return ret;
+	}
+
+	if (!AudioDriver::get_singleton()->try_lock()) {
+		WARN_PRINT("try_lock tried in AudioStreamPlaybackMicrophone.");
+		return ret;
+	}
+
+	unsigned int input_position = AudioDriver::get_singleton()->get_input_position();
+	Vector<int32_t> &buf = AudioDriver::get_singleton()->get_input_buffer();
+	if (input_position < input_ofs)
+		input_position += buf.size();
+	if (p_frames == -1)
+		p_frames = (input_position - input_ofs) / 2;
+	if (input_ofs + p_frames * 2 <= input_position) {
+		ret.resize(p_frames);
+		for (int i = 0; i < p_frames; i++) { // inline of _mix_internal()
+			float l = (buf[input_ofs++] >> 16) / 32768.f;
+			if (input_ofs >= buf.size())
+				input_ofs = 0;
+			float r = (buf[input_ofs++] >> 16) / 32768.f;
+			if (input_ofs >= buf.size())
+				input_ofs = 0;
+			ret.write[i] = Vector2(l, r);
+		}
+	}
+	AudioDriver::get_singleton()->unlock();
+	return ret;
+}
+
+/*
+// not able to implement filling an array, instead having to return an array by value as above
+bool AudioStreamPlaybackMicrophone::mix_microphone(GDExtensionPtr<AudioFrame> p_buffer, int p_frames) {
+	DEV_ASSERT(p_buffer != nullptr);
+	if (!microphone.is_null())
+		return false;
+	unsigned int input_position = AudioDriver::get_singleton()->get_input_position();
+	Vector<int32_t> &buf = AudioDriver::get_singleton()->get_input_buffer();
+	if (input_position < input_ofs)
+		input_position += buf.size();
+	if (input_position < input_ofs + p_frames * 2)
+		return false;
+	int mixed_frames = _mix_internal(p_buffer, p_frames);
+	(void)mixed_frames;
+	DEV_ASSERT(mixed_frames == p_frames);
+	return true;
+}
+*/
+
 int AudioStreamPlaybackMicrophone::mix(AudioFrame *p_buffer, float p_rate_scale, int p_frames) {
 	return AudioStreamPlaybackResampled::mix(p_buffer, p_rate_scale, p_frames);
 }
@@ -458,19 +511,51 @@ void AudioStreamPlaybackMicrophone::start(double p_from_pos) {
 
 	input_ofs = 0;
 
-	if (AudioDriver::get_singleton()->input_start() == OK) {
+	if (AudioDriver::get_singleton()->input_start_count == 0) {
+		uint64_t time_before = OS::get_singleton()->get_ticks_msec();
+		Error err = AudioDriver::get_singleton()->input_start();
+		uint64_t elapsed = (OS::get_singleton()->get_ticks_msec() - time_before);
+		WARN_PRINT(vformat("input microphone started e=%d taking %d ms", err, elapsed));
+		if (err == OK) {
+			active = true;
+			begin_resample();
+			AudioDriver::get_singleton()->input_start_count++;
+		}
+	} else {
 		active = true;
 		begin_resample();
+		AudioDriver::get_singleton()->input_start_count++;
+	}
+}
+
+void AudioStreamPlaybackMicrophone::start_microphone() {
+	if (!microphone.is_null()) {
+		WARN_PRINT("You cannot externally start the microphone on AudioStreamPlaybackMicrophone if it is part of a stream.");
+		return;
 	}
+	start(0.0);
 }
 
 void AudioStreamPlaybackMicrophone::stop() {
 	if (active) {
-		AudioDriver::get_singleton()->input_stop();
+		AudioDriver::get_singleton()->input_start_count--;
+		if (AudioDriver::get_singleton()->input_start_count == 0) {
+			Error err = AudioDriver::get_singleton()->input_stop();
+			(void)err;
+			WARN_PRINT(vformat("input microphone stopped e=%d", err));
+		}
 		active = false;
 	}
 }
 
+void AudioStreamPlaybackMicrophone::stop_microphone() {
+	if (!microphone.is_null()) {
+		WARN_PRINT("You cannot externally stop the microphone on AudioStreamPlaybackMicrophone if it is part of a stream.");
+		return;
+	}
+	stop();
+}
+
 bool AudioStreamPlaybackMicrophone::is_playing() const {
 	return active;
 }
@@ -491,8 +576,17 @@ void AudioStreamPlaybackMicrophone::tag_used_streams() {
 	microphone->tag_used(0);
 }
 
+void AudioStreamPlaybackMicrophone::_bind_methods() {
+	ClassDB::bind_method(D_METHOD("start_microphone"), &AudioStreamPlaybackMicrophone::start_microphone);
+	ClassDB::bind_method(D_METHOD("stop_microphone"), &AudioStreamPlaybackMicrophone::stop_microphone);
+	ClassDB::bind_method(D_METHOD("is_microphone_playing"), &AudioStreamPlaybackMicrophone::is_playing);
+	ClassDB::bind_method(D_METHOD("get_microphone_buffer", "frames"), &AudioStreamPlaybackMicrophone::get_microphone_buffer);
+	//ClassDB::bind_method(D_METHOD("mix_microphone", "p_buffer", "frames"), &AudioStreamPlaybackMicrophone::mix_microphone);
+}
+
 AudioStreamPlaybackMicrophone::~AudioStreamPlaybackMicrophone() {
-	microphone->playbacks.erase(this);
+	if (!microphone.is_null())
+		microphone->playbacks.erase(this);
 	stop();
 }
 

servers/audio/audio_stream.h

@@ -246,6 +246,8 @@ class AudioStreamPlaybackMicrophone : public AudioStreamPlaybackResampled {
 	Ref<AudioStreamMicrophone> microphone;
 
 protected:
+	static void _bind_methods();
+
 	virtual int _mix_internal(AudioFrame *p_buffer, int p_frames) override;
 	virtual float get_stream_sampling_rate() override;
 	virtual double get_playback_position() const override;
@@ -257,6 +259,13 @@ class AudioStreamPlaybackMicrophone : public AudioStreamPlaybackResampled {
 	virtual void stop() override;
 	virtual bool is_playing() const override;
 
+	void start_microphone();
+	void stop_microphone();
+	PackedVector2Array get_microphone_buffer(int p_frames);
+
+	// not able to implement function to fill an existing array
+	// bool mix_microphone(GDExtensionPtr<AudioFrame> p_buffer, int p_frames);
+
 	virtual int get_loop_count() const override; //times it looped
 
 	virtual void seek(double p_time) override;

servers/audio_server.cpp

@@ -102,6 +102,7 @@ void AudioDriver::input_buffer_init(int driver_buffer_frames) {
 	input_size = 0;
 }
 
+// this should only be called from inside a locked block
 void AudioDriver::input_buffer_write(int32_t sample) {
 	if ((int)input_position < input_buffer.size()) {
 		input_buffer.write[input_position++] = sample;
@@ -112,6 +113,7 @@ void AudioDriver::input_buffer_write(int32_t sample) {
 			input_size++;
 		}
 	} else {
+		// this can only happen if this function is called by threads which are not mutually locked
 		WARN_PRINT("input_buffer_write: Invalid input_position=" + itos(input_position) + " input_buffer.size()=" + itos(input_buffer.size()));
 	}
 }