diff --git a/doc/classes/AudioStreamPlaybackMicrophone.xml b/doc/classes/AudioStreamPlaybackMicrophone.xml new file mode 100644 index 000000000000..29c9e38eb68f --- /dev/null +++ b/doc/classes/AudioStreamPlaybackMicrophone.xml @@ -0,0 +1,40 @@ + + + + Playback instance for [AudioStreamMicrophone]. + + + Playback instance for [AudioStreamMicrophone]. When this class is instantiated on its own it can be used to control and extract the microphone audio data directly. + + + + + + + + + Gets the next [param frames] audio samples from the AudioDevice ring buffer. + Returns a [PackedVector2Array] containing exactly [param frames] audio samples if available, or an empty [PackedVector2Array] if insufficient data was available. + The samples are signed floating-point PCM between [code]-1[/code] and [code]1[/code]. You will have to scale them if you want to use them as 8 or 16-bit integer samples. ([code]v = 0x7fff * samples[0].x[/code]) + + + + + + Check if this object has instructed the AudioDevice to record. + + + + + + Instruct the AudioDevice to begin recording if it has not already doing so. + + + + + + Instruct the AudioDevice to stop recording if no other [AudioStreamPlaybackMicrophone] instances are active. + + + + diff --git a/drivers/coreaudio/audio_driver_coreaudio.h b/drivers/coreaudio/audio_driver_coreaudio.h index 54e02514ec4f..395afd89f602 100644 --- a/drivers/coreaudio/audio_driver_coreaudio.h +++ b/drivers/coreaudio/audio_driver_coreaudio.h @@ -118,7 +118,7 @@ class AudioDriverCoreAudio : public AudioDriver { virtual Error input_start() override; virtual Error input_stop() override; - bool try_lock(); + virtual bool try_lock() override; void stop(); AudioDriverCoreAudio(); diff --git a/drivers/pulseaudio/audio_driver_pulseaudio.cpp b/drivers/pulseaudio/audio_driver_pulseaudio.cpp index 669e6c2aa951..ee2f75dbeb82 100644 --- a/drivers/pulseaudio/audio_driver_pulseaudio.cpp +++ b/drivers/pulseaudio/audio_driver_pulseaudio.cpp @@ -537,6 +537,7 @@ void AudioDriverPulseAudio::thread_func(void *p_udata) { ERR_PRINT("pa_stream_peek error"); } else { int16_t *srcptr = (int16_t *)ptr; + ad->lock(); for (size_t i = bytes >> 1; i > 0; i--) { int32_t sample = int32_t(*srcptr++) << 16; ad->input_buffer_write(sample); @@ -546,6 +547,7 @@ void AudioDriverPulseAudio::thread_func(void *p_udata) { ad->input_buffer_write(sample); } } + ad->unlock(); read_bytes += bytes; ret = pa_stream_drop(ad->pa_rec_str); diff --git a/drivers/wasapi/audio_driver_wasapi.cpp b/drivers/wasapi/audio_driver_wasapi.cpp index b5cb8da249af..75be480622b8 100644 --- a/drivers/wasapi/audio_driver_wasapi.cpp +++ b/drivers/wasapi/audio_driver_wasapi.cpp @@ -848,6 +848,7 @@ void AudioDriverWASAPI::thread_func(void *p_udata) { ERR_BREAK(hr != S_OK); // fixme: Only works for floating point atm + ad->lock(); for (UINT32 j = 0; j < num_frames_available; j++) { int32_t l, r; @@ -868,6 +869,7 @@ void AudioDriverWASAPI::thread_func(void *p_udata) { ad->input_buffer_write(l); ad->input_buffer_write(r); } + ad->unlock(); read_frames += num_frames_available; diff --git a/platform/android/audio_driver_opensl.cpp b/platform/android/audio_driver_opensl.cpp index ef9c51db079b..d6b7fa102dc0 100644 --- a/platform/android/audio_driver_opensl.cpp +++ b/platform/android/audio_driver_opensl.cpp @@ -184,11 +184,13 @@ void AudioDriverOpenSL::start() { } void AudioDriverOpenSL::_record_buffer_callback(SLAndroidSimpleBufferQueueItf queueItf) { + lock(); for (int i = 0; i < rec_buffer.size(); i++) { int32_t sample = rec_buffer[i] << 16; input_buffer_write(sample); input_buffer_write(sample); // call twice to convert to Stereo } + unlock(); SLresult res = (*recordBufferQueueItf)->Enqueue(recordBufferQueueItf, rec_buffer.ptrw(), rec_buffer.size() * sizeof(int16_t)); ERR_FAIL_COND(res != SL_RESULT_SUCCESS); @@ -272,29 +274,30 @@ Error AudioDriverOpenSL::input_start() { return ERR_ALREADY_IN_USE; } - if (OS::get_singleton()->request_permission("RECORD_AUDIO")) { - return init_input_device(); + if (!OS::get_singleton()->request_permission("RECORD_AUDIO")) { + WARN_PRINT("Unable to start audio capture - No RECORD_AUDIO permission -- yet"); + return ERR_UNAUTHORIZED; } - - WARN_PRINT("Unable to start audio capture - No RECORD_AUDIO permission"); - return ERR_UNAUTHORIZED; + return init_input_device(); } Error AudioDriverOpenSL::input_stop() { - if (!recordItf || !recordBufferQueueItf) { - return ERR_CANT_OPEN; + //if (!recordItf || !recordBufferQueueItf) { + // return ERR_CANT_OPEN; + //} + if (recordItf) { + (*recordItf)->SetRecordState(recordItf, SL_RECORDSTATE_STOPPED); + recordItf = nullptr; } - SLuint32 state; - SLresult res = (*recordItf)->GetRecordState(recordItf, &state); - ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN); - - if (state != SL_RECORDSTATE_STOPPED) { - res = (*recordItf)->SetRecordState(recordItf, SL_RECORDSTATE_STOPPED); - ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN); + if (recordBufferQueueItf) { + (*recordBufferQueueItf)->Clear(recordBufferQueueItf); + recordBufferQueueItf = nullptr; + } - res = (*recordBufferQueueItf)->Clear(recordBufferQueueItf); - ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN); + if (recorder) { + (*recorder)->Destroy(recorder); + recorder = nullptr; } return OK; @@ -314,6 +317,13 @@ void AudioDriverOpenSL::lock() { } } +bool AudioDriverOpenSL::try_lock() { + if (active) { + return mutex.try_lock(); + } + return true; +} + void AudioDriverOpenSL::unlock() { if (active) { mutex.unlock(); diff --git a/platform/android/audio_driver_opensl.h b/platform/android/audio_driver_opensl.h index bcd173826acb..d1c96d0b7467 100644 --- a/platform/android/audio_driver_opensl.h +++ b/platform/android/audio_driver_opensl.h @@ -97,6 +97,7 @@ class AudioDriverOpenSL : public AudioDriver { virtual SpeakerMode get_speaker_mode() const override; virtual void lock() override; + virtual bool try_lock() override; virtual void unlock() override; virtual void finish() override; diff --git a/platform/android/java_godot_lib_jni.cpp b/platform/android/java_godot_lib_jni.cpp index 5c1e78dcc42e..0c0c9e5a4d04 100644 --- a/platform/android/java_godot_lib_jni.cpp +++ b/platform/android/java_godot_lib_jni.cpp @@ -566,9 +566,12 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_filePickerCallback(JN JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_requestPermissionResult(JNIEnv *env, jclass clazz, jstring p_permission, jboolean p_result) { String permission = jstring_to_string(p_permission, env); - if (permission == "android.permission.RECORD_AUDIO" && p_result) { - AudioDriver::get_singleton()->input_start(); - } + + // this code block is not helpful due to the lack of error handling and the + // information not getting back to the AudioStreamPlaybackMicrophone + //if (permission == "android.permission.RECORD_AUDIO" && p_result) { + // AudioDriver::get_singleton()->input_start(); + //} if (os_android->get_main_loop()) { os_android->get_main_loop()->emit_signal(SNAME("on_request_permissions_result"), permission, p_result == JNI_TRUE); diff --git a/platform/web/audio_driver_web.cpp b/platform/web/audio_driver_web.cpp index f3d5b5cd1a16..a5064cd6a1c8 100644 --- a/platform/web/audio_driver_web.cpp +++ b/platform/web/audio_driver_web.cpp @@ -105,6 +105,8 @@ void AudioDriverWeb::_audio_driver_capture(int p_from, int p_samples) { if (to_read == 0) { to_read = max_samples; } + + lock(); // High part if (read_pos + to_read > max_samples) { const int samples_high = max_samples - read_pos; @@ -118,6 +120,7 @@ void AudioDriverWeb::_audio_driver_capture(int p_from, int p_samples) { for (int i = read_pos; i < read_pos + to_read; i++) { input_buffer_write(int32_t(input_rb[i] * 32768.f) * (1U << 16)); } + unlock(); } Error AudioDriverWeb::init() { diff --git a/servers/audio/audio_stream.cpp b/servers/audio/audio_stream.cpp index 70f005930bbf..29af7ee5102a 100644 --- a/servers/audio/audio_stream.cpp +++ b/servers/audio/audio_stream.cpp @@ -390,7 +390,7 @@ bool AudioStreamMicrophone::is_monophonic() const { int AudioStreamPlaybackMicrophone::_mix_internal(AudioFrame *p_buffer, int p_frames) { AudioDriver::get_singleton()->lock(); - Vector buf = AudioDriver::get_singleton()->get_input_buffer(); + Vector &buf = AudioDriver::get_singleton()->get_input_buffer(); unsigned int input_size = AudioDriver::get_singleton()->get_input_size(); int mix_rate = AudioDriver::get_singleton()->get_input_mix_rate(); unsigned int playback_delay = MIN(((50 * mix_rate) / 1000) * 2, buf.size() >> 1); @@ -438,6 +438,59 @@ int AudioStreamPlaybackMicrophone::_mix_internal(AudioFrame *p_buffer, int p_fra return mixed_frames; } +PackedVector2Array AudioStreamPlaybackMicrophone::get_microphone_buffer(int p_frames) { + PackedVector2Array ret; + if (!microphone.is_null()) { + WARN_PRINT("You cannot call get_microphone_buffer() on AudioStreamPlaybackMicrophone if it is part of a stream."); + return ret; + } + + if (!AudioDriver::get_singleton()->try_lock()) { + WARN_PRINT("try_lock tried in AudioStreamPlaybackMicrophone."); + return ret; + } + + unsigned int input_position = AudioDriver::get_singleton()->get_input_position(); + Vector &buf = AudioDriver::get_singleton()->get_input_buffer(); + if (input_position < input_ofs) + input_position += buf.size(); + if (p_frames == -1) + p_frames = (input_position - input_ofs) / 2; + if (input_ofs + p_frames * 2 <= input_position) { + ret.resize(p_frames); + for (int i = 0; i < p_frames; i++) { // inline of _mix_internal() + float l = (buf[input_ofs++] >> 16) / 32768.f; + if (input_ofs >= buf.size()) + input_ofs = 0; + float r = (buf[input_ofs++] >> 16) / 32768.f; + if (input_ofs >= buf.size()) + input_ofs = 0; + ret.write[i] = Vector2(l, r); + } + } + AudioDriver::get_singleton()->unlock(); + return ret; +} + +/* +// not able to implement filling an array, instead having to return an array by value as above +bool AudioStreamPlaybackMicrophone::mix_microphone(GDExtensionPtr p_buffer, int p_frames) { + DEV_ASSERT(p_buffer != nullptr); + if (!microphone.is_null()) + return false; + unsigned int input_position = AudioDriver::get_singleton()->get_input_position(); + Vector &buf = AudioDriver::get_singleton()->get_input_buffer(); + if (input_position < input_ofs) + input_position += buf.size(); + if (input_position < input_ofs + p_frames * 2) + return false; + int mixed_frames = _mix_internal(p_buffer, p_frames); + (void)mixed_frames; + DEV_ASSERT(mixed_frames == p_frames); + return true; +} +*/ + int AudioStreamPlaybackMicrophone::mix(AudioFrame *p_buffer, float p_rate_scale, int p_frames) { return AudioStreamPlaybackResampled::mix(p_buffer, p_rate_scale, p_frames); } @@ -458,19 +511,51 @@ void AudioStreamPlaybackMicrophone::start(double p_from_pos) { input_ofs = 0; - if (AudioDriver::get_singleton()->input_start() == OK) { + if (AudioDriver::get_singleton()->input_start_count == 0) { + uint64_t time_before = OS::get_singleton()->get_ticks_msec(); + Error err = AudioDriver::get_singleton()->input_start(); + uint64_t elapsed = (OS::get_singleton()->get_ticks_msec() - time_before); + WARN_PRINT(vformat("input microphone started e=%d taking %d ms", err, elapsed)); + if (err == OK) { + active = true; + begin_resample(); + AudioDriver::get_singleton()->input_start_count++; + } + } else { active = true; begin_resample(); + AudioDriver::get_singleton()->input_start_count++; + } +} + +void AudioStreamPlaybackMicrophone::start_microphone() { + if (!microphone.is_null()) { + WARN_PRINT("You cannot externally start the microphone on AudioStreamPlaybackMicrophone if it is part of a stream."); + return; } + start(0.0); } void AudioStreamPlaybackMicrophone::stop() { if (active) { - AudioDriver::get_singleton()->input_stop(); + AudioDriver::get_singleton()->input_start_count--; + if (AudioDriver::get_singleton()->input_start_count == 0) { + Error err = AudioDriver::get_singleton()->input_stop(); + (void)err; + WARN_PRINT(vformat("input microphone stopped e=%d", err)); + } active = false; } } +void AudioStreamPlaybackMicrophone::stop_microphone() { + if (!microphone.is_null()) { + WARN_PRINT("You cannot externally stop the microphone on AudioStreamPlaybackMicrophone if it is part of a stream."); + return; + } + stop(); +} + bool AudioStreamPlaybackMicrophone::is_playing() const { return active; } @@ -491,8 +576,17 @@ void AudioStreamPlaybackMicrophone::tag_used_streams() { microphone->tag_used(0); } +void AudioStreamPlaybackMicrophone::_bind_methods() { + ClassDB::bind_method(D_METHOD("start_microphone"), &AudioStreamPlaybackMicrophone::start_microphone); + ClassDB::bind_method(D_METHOD("stop_microphone"), &AudioStreamPlaybackMicrophone::stop_microphone); + ClassDB::bind_method(D_METHOD("is_microphone_playing"), &AudioStreamPlaybackMicrophone::is_playing); + ClassDB::bind_method(D_METHOD("get_microphone_buffer", "frames"), &AudioStreamPlaybackMicrophone::get_microphone_buffer); + //ClassDB::bind_method(D_METHOD("mix_microphone", "p_buffer", "frames"), &AudioStreamPlaybackMicrophone::mix_microphone); +} + AudioStreamPlaybackMicrophone::~AudioStreamPlaybackMicrophone() { - microphone->playbacks.erase(this); + if (!microphone.is_null()) + microphone->playbacks.erase(this); stop(); } diff --git a/servers/audio/audio_stream.h b/servers/audio/audio_stream.h index 65efccdc28e7..9a35ea1e1eb4 100644 --- a/servers/audio/audio_stream.h +++ b/servers/audio/audio_stream.h @@ -246,6 +246,8 @@ class AudioStreamPlaybackMicrophone : public AudioStreamPlaybackResampled { Ref microphone; protected: + static void _bind_methods(); + virtual int _mix_internal(AudioFrame *p_buffer, int p_frames) override; virtual float get_stream_sampling_rate() override; virtual double get_playback_position() const override; @@ -257,6 +259,13 @@ class AudioStreamPlaybackMicrophone : public AudioStreamPlaybackResampled { virtual void stop() override; virtual bool is_playing() const override; + void start_microphone(); + void stop_microphone(); + PackedVector2Array get_microphone_buffer(int p_frames); + + // not able to implement function to fill an existing array + // bool mix_microphone(GDExtensionPtr p_buffer, int p_frames); + virtual int get_loop_count() const override; //times it looped virtual void seek(double p_time) override; diff --git a/servers/audio_server.cpp b/servers/audio_server.cpp index 63b2f28e7415..cedb729343ec 100644 --- a/servers/audio_server.cpp +++ b/servers/audio_server.cpp @@ -102,6 +102,7 @@ void AudioDriver::input_buffer_init(int driver_buffer_frames) { input_size = 0; } +// this should only be called from inside a locked block void AudioDriver::input_buffer_write(int32_t sample) { if ((int)input_position < input_buffer.size()) { input_buffer.write[input_position++] = sample; @@ -112,6 +113,7 @@ void AudioDriver::input_buffer_write(int32_t sample) { input_size++; } } else { + // this can only happen if this function is called by threads which are not mutually locked WARN_PRINT("input_buffer_write: Invalid input_position=" + itos(input_position) + " input_buffer.size()=" + itos(input_buffer.size())); } } diff --git a/servers/audio_server.h b/servers/audio_server.h index 9a80882dd7a1..6bdd0297c6d5 100644 --- a/servers/audio_server.h +++ b/servers/audio_server.h @@ -58,10 +58,13 @@ class AudioDriver { SafeNumeric prof_time; #endif + friend class AudioStreamPlaybackMicrophone; + protected: Vector input_buffer; unsigned int input_position = 0; unsigned int input_size = 0; + int32_t input_start_count = 0; void audio_server_process(int p_frames, int32_t *p_buffer, bool p_update_mix_time = true); void update_mix_time(int p_frames); @@ -104,6 +107,10 @@ class AudioDriver { virtual float get_latency() { return 0; } virtual void lock() = 0; + virtual bool try_lock() { + lock(); + return true; + } virtual void unlock() = 0; virtual void finish() = 0; @@ -123,7 +130,7 @@ class AudioDriver { SpeakerMode get_speaker_mode_by_total_channels(int p_channels) const; int get_total_channels_by_speaker_mode(SpeakerMode) const; - Vector get_input_buffer() { return input_buffer; } + Vector &get_input_buffer() { return input_buffer; } unsigned int get_input_position() { return input_position; } unsigned int get_input_size() { return input_size; } diff --git a/servers/register_server_types.cpp b/servers/register_server_types.cpp index 3e0caeea6594..670ebd55bd61 100644 --- a/servers/register_server_types.cpp +++ b/servers/register_server_types.cpp @@ -160,6 +160,7 @@ void register_server_types() { GDREGISTER_CLASS(AudioStreamPlayback); GDREGISTER_VIRTUAL_CLASS(AudioStreamPlaybackResampled); GDREGISTER_CLASS(AudioStreamMicrophone); + GDREGISTER_CLASS(AudioStreamPlaybackMicrophone); GDREGISTER_CLASS(AudioStreamRandomizer); GDREGISTER_CLASS(AudioSample); GDREGISTER_CLASS(AudioSamplePlayback);